Autopsy  4.18.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestEventsListener.java
Go to the documentation of this file.
1 /*
2  * Central Repository
3  *
4  * Copyright 2017-2020 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.centralrepository.eventlisteners;
20 
21 import com.google.common.util.concurrent.ThreadFactoryBuilder;
22 import java.beans.PropertyChangeEvent;
23 import java.beans.PropertyChangeListener;
24 import static java.lang.Boolean.FALSE;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.Collection;
28 import java.util.EnumSet;
29 import java.util.LinkedHashSet;
30 import java.util.List;
31 import java.util.Set;
32 import java.util.concurrent.ExecutorService;
33 import java.util.concurrent.Executors;
34 import java.util.logging.Level;
35 import java.util.stream.Collectors;
36 import org.apache.commons.lang3.StringUtils;
37 import org.openide.util.NbBundle;
49 import org.sleuthkit.datamodel.AbstractFile;
50 import org.sleuthkit.datamodel.Blackboard;
51 import org.sleuthkit.datamodel.BlackboardArtifact;
52 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT;
53 import org.sleuthkit.datamodel.BlackboardAttribute;
56 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT;
57 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT;
58 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME;
60 import org.sleuthkit.datamodel.Content;
61 import org.sleuthkit.datamodel.Image;
62 import org.sleuthkit.datamodel.SleuthkitCase;
63 import org.sleuthkit.datamodel.TskCoreException;
65 import org.sleuthkit.datamodel.Score;
66 
71 @NbBundle.Messages({"IngestEventsListener.ingestmodule.name=Central Repository"})
72 public class IngestEventsListener {
73  private static final Logger LOGGER = Logger.getLogger(CorrelationAttributeInstance.class.getName());
74  private static final Set<IngestManager.IngestJobEvent> INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED);
75  private static final Set<IngestManager.IngestModuleEvent> INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(DATA_ADDED);
76  private static final String MODULE_NAME = Bundle.IngestEventsListener_ingestmodule_name();
77  private static int correlationModuleInstanceCount;
78  private static boolean flagNotableItems;
79  private static boolean flagSeenDevices;
80  private static boolean createCrProperties;
81  private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d";
82  private final ExecutorService jobProcessingExecutor;
83  private final PropertyChangeListener pcl1 = new IngestModuleEventListener();
84  private final PropertyChangeListener pcl2 = new IngestJobEventListener();
85  final Collection<String> recentlyAddedCeArtifacts = new LinkedHashSet<>();
86 
88  jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(INGEST_EVENT_THREAD_NAME).build());
89  }
90 
91  public void shutdown() {
92  ThreadUtils.shutDownTaskExecutor(jobProcessingExecutor);
93  }
94 
95  /*
96  * Add all of our Ingest Event Listeners to the IngestManager Instance.
97  */
98  public void installListeners() {
99  IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl1);
100  IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl2);
101  }
102 
103  /*
104  * Remove all of our Ingest Event Listeners from the IngestManager Instance.
105  */
106  public void uninstallListeners() {
109  }
110 
115  public synchronized static void incrementCorrelationEngineModuleCount() {
116  correlationModuleInstanceCount++; //Should be called once in the Central Repository module's startup method.
117  }
118 
123  public synchronized static void decrementCorrelationEngineModuleCount() {
124  if (getCeModuleInstanceCount() > 0) { //prevent it ingestJobCounter from going negative
125  correlationModuleInstanceCount--; //Should be called once in the Central Repository module's shutdown method.
126  }
127  }
128 
133  synchronized static void resetCeModuleInstanceCount() {
134  correlationModuleInstanceCount = 0; //called when a case is opened in case for some reason counter was not reset
135  }
136 
143  public synchronized static int getCeModuleInstanceCount() {
144  return correlationModuleInstanceCount;
145  }
146 
152  public synchronized static boolean isFlagNotableItems() {
153  return flagNotableItems;
154  }
155 
161  public synchronized static boolean isFlagSeenDevices() {
162  return flagSeenDevices;
163  }
164 
170  public synchronized static boolean shouldCreateCrProperties() {
171  return createCrProperties;
172  }
173 
179  public synchronized static void setFlagNotableItems(boolean value) {
180  flagNotableItems = value;
181  }
182 
188  public synchronized static void setFlagSeenDevices(boolean value) {
189  flagSeenDevices = value;
190  }
191 
197  public synchronized static void setCreateCrProperties(boolean value) {
198  createCrProperties = value;
199  }
200 
209  @NbBundle.Messages({"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
210  "IngestEventsListener.prevCaseComment.text=Previous Case: "})
211  static private void makeAndPostPreviousNotableArtifact(BlackboardArtifact originalArtifact, List<String> caseDisplayNames) {
212  Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(
213  new BlackboardAttribute(
214  TSK_SET_NAME, MODULE_NAME,
215  Bundle.IngestEventsListener_prevTaggedSet_text()),
216  new BlackboardAttribute(
217  TSK_COMMENT, MODULE_NAME,
218  Bundle.IngestEventsListener_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(","))),
219  new BlackboardAttribute(
220  TSK_ASSOCIATED_ARTIFACT, MODULE_NAME,
221  originalArtifact.getArtifactID()));
222  makeAndPostInterestingArtifact(originalArtifact, attributesForNewArtifact, Bundle.IngestEventsListener_prevTaggedSet_text());
223  }
224 
233  @NbBundle.Messages({"IngestEventsListener.prevExists.text=Previously Seen Devices (Central Repository)",
234  "# {0} - typeName",
235  "# {1} - count",
236  "IngestEventsListener.prevCount.text=Number of previous {0}: {1}"})
237  static private void makeAndPostPreviousSeenArtifact(BlackboardArtifact originalArtifact, List<String> caseDisplayNames) {
238  Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(new BlackboardAttribute(
239  TSK_SET_NAME, MODULE_NAME,
240  Bundle.IngestEventsListener_prevExists_text()),
241  new BlackboardAttribute(
242  TSK_COMMENT, MODULE_NAME,
243  Bundle.IngestEventsListener_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(","))),
244  new BlackboardAttribute(
245  TSK_ASSOCIATED_ARTIFACT, MODULE_NAME,
246  originalArtifact.getArtifactID()));
247  makeAndPostInterestingArtifact(originalArtifact, attributesForNewArtifact, Bundle.IngestEventsListener_prevExists_text());
248  }
249 
250 
259  private static void makeAndPostInterestingArtifact(BlackboardArtifact originalArtifact, Collection<BlackboardAttribute> attributesForNewArtifact, String configuration) {
260  try {
261  SleuthkitCase tskCase = originalArtifact.getSleuthkitCase();
262  AbstractFile abstractFile = tskCase.getAbstractFileById(originalArtifact.getObjectID());
263  Blackboard blackboard = tskCase.getBlackboard();
264  // Create artifact if it doesn't already exist.
265  if (!blackboard.artifactExists(abstractFile, TSK_INTERESTING_ARTIFACT_HIT, attributesForNewArtifact)) {
266  BlackboardArtifact newInterestingArtifact = abstractFile.newAnalysisResult(
267  BlackboardArtifact.Type.TSK_INTERESTING_ARTIFACT_HIT, Score.SCORE_LIKELY_NOTABLE,
268  null, configuration, null, attributesForNewArtifact)
269  .getAnalysisResult();
270 
271  try {
272  // index the artifact for keyword search
273  blackboard.postArtifact(newInterestingArtifact, MODULE_NAME);
274  } catch (Blackboard.BlackboardException ex) {
275  LOGGER.log(Level.SEVERE, "Unable to index blackboard artifact " + newInterestingArtifact.getArtifactID(), ex); //NON-NLS
276  }
277  }
278  } catch (TskCoreException ex) {
279  LOGGER.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS
280  } catch (IllegalStateException ex) {
281  LOGGER.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
282  }
283  }
284 
285  private class IngestModuleEventListener implements PropertyChangeListener {
286 
287  @Override
288  public void propertyChange(PropertyChangeEvent evt) {
289  //if ingest is running we want there to check if there is a Central Repository module running
290  //sometimes artifacts are generated by DSPs or other sources while ingest is not running
291  //in these cases we still want to create correlation attributesForNewArtifact for those artifacts when appropriate
292  if (!IngestManager.getInstance().isIngestRunning() || getCeModuleInstanceCount() > 0) {
293  CentralRepository dbManager;
294  try {
295  dbManager = CentralRepository.getInstance();
296  } catch (CentralRepoException ex) {
297  LOGGER.log(Level.SEVERE, "Failed to connect to Central Repository database.", ex);
298  return;
299  }
300  switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) {
301  case DATA_ADDED: {
302  //if ingest isn't running create the interesting items otherwise use the ingest module setting to determine if we create interesting items
303  boolean flagNotable = !IngestManager.getInstance().isIngestRunning() || isFlagNotableItems();
304  boolean flagPrevious = !IngestManager.getInstance().isIngestRunning() || isFlagSeenDevices();
305  boolean createAttributes = !IngestManager.getInstance().isIngestRunning() || shouldCreateCrProperties();
306  jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable, flagPrevious, createAttributes));
307  break;
308  }
309  default:
310  break;
311  }
312  }
313  }
314  }
315 
316  private class IngestJobEventListener implements PropertyChangeListener {
317 
318  @Override
319  public void propertyChange(PropertyChangeEvent evt) {
320  CentralRepository dbManager;
321  try {
322  dbManager = CentralRepository.getInstance();
323  } catch (CentralRepoException ex) {
324  LOGGER.log(Level.SEVERE, "Failed to connect to Central Repository database.", ex);
325  return;
326  }
327 
328  switch (IngestManager.IngestJobEvent.valueOf(evt.getPropertyName())) {
329  case DATA_SOURCE_ANALYSIS_COMPLETED: {
330  jobProcessingExecutor.submit(new AnalysisCompleteTask(dbManager, evt));
331  break;
332  }
333  default:
334  break;
335  }
336  }
337 
338  }
339 
340  private final class AnalysisCompleteTask implements Runnable {
341 
343  private final PropertyChangeEvent event;
344 
345  private AnalysisCompleteTask(CentralRepository db, PropertyChangeEvent evt) {
346  dbManager = db;
347  event = evt;
348  }
349 
350  @Override
351  public void run() {
352  // clear the tracker to reduce memory usage
353  if (getCeModuleInstanceCount() == 0) {
354  recentlyAddedCeArtifacts.clear();
355  }
356  //else another instance of the Central Repository Module is still being run.
357 
358  /*
359  * Ensure the data source in the Central Repository has hash values
360  * that match those in the case database.
361  */
362  if (!CentralRepository.isEnabled()) {
363  return;
364  }
365  Content dataSource;
366  String dataSourceName = "";
367  long dataSourceObjectId = -1;
368  try {
369  dataSource = ((DataSourceAnalysisEvent) event).getDataSource();
370  /*
371  * We only care about Images for the purpose of
372  * updating hash values.
373  */
374  if (!(dataSource instanceof Image)) {
375  return;
376  }
377 
378  dataSourceName = dataSource.getName();
379  dataSourceObjectId = dataSource.getId();
380 
381  Case openCase = Case.getCurrentCaseThrows();
382 
383  CorrelationCase correlationCase = dbManager.getCase(openCase);
384  if (null == correlationCase) {
385  correlationCase = dbManager.newCase(openCase);
386  }
387 
388  CorrelationDataSource correlationDataSource = dbManager.getDataSource(correlationCase, dataSource.getId());
389  if (correlationDataSource == null) {
390  // Add the data source.
391  CorrelationDataSource.fromTSKDataSource(correlationCase, dataSource);
392  } else {
393  // Sync the data source hash values if necessary.
394  if (dataSource instanceof Image) {
395  Image image = (Image) dataSource;
396 
397  String imageMd5Hash = image.getMd5();
398  if (imageMd5Hash == null) {
399  imageMd5Hash = "";
400  }
401  String crMd5Hash = correlationDataSource.getMd5();
402  if (StringUtils.equals(imageMd5Hash, crMd5Hash) == false) {
403  correlationDataSource.setMd5(imageMd5Hash);
404  }
405 
406  String imageSha1Hash = image.getSha1();
407  if (imageSha1Hash == null) {
408  imageSha1Hash = "";
409  }
410  String crSha1Hash = correlationDataSource.getSha1();
411  if (StringUtils.equals(imageSha1Hash, crSha1Hash) == false) {
412  correlationDataSource.setSha1(imageSha1Hash);
413  }
414 
415  String imageSha256Hash = image.getSha256();
416  if (imageSha256Hash == null) {
417  imageSha256Hash = "";
418  }
419  String crSha256Hash = correlationDataSource.getSha256();
420  if (StringUtils.equals(imageSha256Hash, crSha256Hash) == false) {
421  correlationDataSource.setSha256(imageSha256Hash);
422  }
423  }
424  }
425  } catch (CentralRepoException ex) {
426  LOGGER.log(Level.SEVERE, String.format(
427  "Unable to fetch data from the Central Repository for data source '%s' (obj_id=%d)",
428  dataSourceName, dataSourceObjectId), ex);
429  } catch (NoCurrentCaseException ex) {
430  LOGGER.log(Level.SEVERE, "No current case opened.", ex);
431  } catch (TskCoreException ex) {
432  LOGGER.log(Level.SEVERE, String.format(
433  "Unable to fetch data from the case database for data source '%s' (obj_id=%d)",
434  dataSourceName, dataSourceObjectId), ex);
435  }
436  } // DATA_SOURCE_ANALYSIS_COMPLETED
437  }
438 
439  private final class DataAddedTask implements Runnable {
440 
442  private final PropertyChangeEvent event;
443  private final boolean flagNotableItemsEnabled;
444  private final boolean flagPreviousItemsEnabled;
445  private final boolean createCorrelationAttributes;
446 
447  private DataAddedTask(CentralRepository db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled, boolean createCorrelationAttributes) {
448  this.dbManager = db;
449  this.event = evt;
450  this.flagNotableItemsEnabled = flagNotableItemsEnabled;
451  this.flagPreviousItemsEnabled = flagPreviousItemsEnabled;
452  this.createCorrelationAttributes = createCorrelationAttributes;
453  }
454 
455  @Override
456  public void run() {
457  if (!CentralRepository.isEnabled()) {
458  return;
459  }
460  final ModuleDataEvent mde = (ModuleDataEvent) event.getOldValue();
461  Collection<BlackboardArtifact> bbArtifacts = mde.getArtifacts();
462  if (null == bbArtifacts) { //the ModuleDataEvents don't always have a collection of artifacts set
463  return;
464  }
465  List<CorrelationAttributeInstance> eamArtifacts = new ArrayList<>();
466 
467  for (BlackboardArtifact bbArtifact : bbArtifacts) {
468  // makeCorrAttrToSave will filter out artifacts which should not be sources of CR data.
469  List<CorrelationAttributeInstance> convertedArtifacts = CorrelationAttributeUtil.makeCorrAttrsToSave(bbArtifact);
470  for (CorrelationAttributeInstance eamArtifact : convertedArtifacts) {
471  try {
472  // Only do something with this artifact if it's unique within the job
473  if (recentlyAddedCeArtifacts.add(eamArtifact.toString())) {
474  // Was it previously marked as bad?
475  // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad".
476  // if getKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case,
477  // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB.
478  if (flagNotableItemsEnabled) {
479  List<String> caseDisplayNames;
480  try {
481  caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
482  if (!caseDisplayNames.isEmpty()) {
483  makeAndPostPreviousNotableArtifact(bbArtifact,
484  caseDisplayNames);
485  }
487  LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex);
488  }
489  }
490  if (flagPreviousItemsEnabled
491  && (eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.USBID_TYPE_ID
492  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.ICCID_TYPE_ID
493  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.IMEI_TYPE_ID
494  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.IMSI_TYPE_ID
495  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.MAC_TYPE_ID)) {
496  try {
497  //only alert to previous instances when they were in another case
498  List<CorrelationAttributeInstance> previousOccurences = dbManager.getArtifactInstancesByTypeValue(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
499  List<String> caseDisplayNames;
500  for (CorrelationAttributeInstance instance : previousOccurences) {
501  if (!instance.getCorrelationCase().getCaseUUID().equals(eamArtifact.getCorrelationCase().getCaseUUID())) {
502  caseDisplayNames = dbManager.getListCasesHavingArtifactInstances(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
503  makeAndPostPreviousSeenArtifact(bbArtifact, caseDisplayNames);
504  break;
505  }
506  }
508  LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex);
509  }
510  }
511  if (createCorrelationAttributes) {
512  eamArtifacts.add(eamArtifact);
513  }
514  }
515  } catch (CentralRepoException ex) {
516  LOGGER.log(Level.SEVERE, "Error counting notable artifacts.", ex);
517  }
518  }
519  }
520  if (FALSE == eamArtifacts.isEmpty()) {
521  for (CorrelationAttributeInstance eamArtifact : eamArtifacts) {
522  try {
523  dbManager.addArtifactInstance(eamArtifact);
524  } catch (CentralRepoException ex) {
525  LOGGER.log(Level.SEVERE, "Error adding artifact to database.", ex); //NON-NLS
526  }
527  }
528  } // DATA_ADDED
529  }
530  }
531 }
Collection< BlackboardArtifact > getArtifacts()
void removeIngestModuleEventListener(final PropertyChangeListener listener)
static synchronized IngestManager getInstance()
List< String > getListCasesHavingArtifactInstances(CorrelationAttributeInstance.Type aType, String value)
static void makeAndPostPreviousSeenArtifact(BlackboardArtifact originalArtifact, List< String > caseDisplayNames)
List< CorrelationAttributeInstance > getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value)
static CorrelationDataSource fromTSKDataSource(CorrelationCase correlationCase, Content dataSource)
List< String > getListCasesHavingArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value)
void removeIngestJobEventListener(final PropertyChangeListener listener)
void addArtifactInstance(CorrelationAttributeInstance eamArtifact)
static void shutDownTaskExecutor(ExecutorService executor)
static void makeAndPostInterestingArtifact(BlackboardArtifact originalArtifact, Collection< BlackboardAttribute > attributesForNewArtifact, String configuration)
void addIngestJobEventListener(final PropertyChangeListener listener)
static void makeAndPostPreviousNotableArtifact(BlackboardArtifact originalArtifact, List< String > caseDisplayNames)
static List< CorrelationAttributeInstance > makeCorrAttrsToSave(BlackboardArtifact artifact)
void addIngestModuleEventListener(final PropertyChangeListener listener)
CorrelationDataSource getDataSource(CorrelationCase correlationCase, Long caseDbDataSourceId)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
DataAddedTask(CentralRepository db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled, boolean createCorrelationAttributes)

Copyright © 2012-2021 Basis Technology. Generated on: Thu Jul 8 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.