Autopsy  4.9.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestEventsListener.java
Go to the documentation of this file.
1 /*
2  * Central Repository
3  *
4  * Copyright 2015-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.centralrepository.eventlisteners;
20 
21 import com.google.common.util.concurrent.ThreadFactoryBuilder;
22 import java.beans.PropertyChangeEvent;
23 import java.beans.PropertyChangeListener;
24 import static java.lang.Boolean.FALSE;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.LinkedHashSet;
28 import java.util.List;
29 import java.util.concurrent.ExecutorService;
30 import java.util.concurrent.Executors;
31 import java.util.logging.Level;
32 import java.util.stream.Collectors;
33 import org.openide.util.NbBundle;
45 import org.sleuthkit.datamodel.AbstractFile;
46 import org.sleuthkit.datamodel.BlackboardArtifact;
47 import org.sleuthkit.datamodel.BlackboardAttribute;
48 import org.sleuthkit.datamodel.TskCoreException;
51 import org.sleuthkit.datamodel.SleuthkitCase;
52 
57 public class IngestEventsListener {
58 
59  private static final Logger LOGGER = Logger.getLogger(CorrelationAttributeInstance.class.getName());
60 
61  final Collection<String> recentlyAddedCeArtifacts = new LinkedHashSet<>();
62  private static int correlationModuleInstanceCount;
63  private static boolean flagNotableItems;
64  private static boolean flagSeenDevices;
65  private static boolean createCrProperties;
66  private final ExecutorService jobProcessingExecutor;
67  private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d";
68  private final PropertyChangeListener pcl1 = new IngestModuleEventListener();
69  private final PropertyChangeListener pcl2 = new IngestJobEventListener();
70 
72  jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(INGEST_EVENT_THREAD_NAME).build());
73  }
74 
75  void shutdown() {
76  ThreadUtils.shutDownTaskExecutor(jobProcessingExecutor);
77  }
78 
79  /*
80  * Add all of our Ingest Event Listeners to the IngestManager Instance.
81  */
82  public void installListeners() {
85  }
86 
87  /*
88  * Remove all of our Ingest Event Listeners from the IngestManager Instance.
89  */
90  public void uninstallListeners() {
93  }
94 
99  public synchronized static void incrementCorrelationEngineModuleCount() {
100  correlationModuleInstanceCount++; //Should be called once in the Correlation Engine module's startup method.
101  }
102 
107  public synchronized static void decrementCorrelationEngineModuleCount() {
108  if (getCeModuleInstanceCount() > 0) { //prevent it ingestJobCounter from going negative
109  correlationModuleInstanceCount--; //Should be called once in the Correlation Engine module's shutdown method.
110  }
111  }
112 
117  synchronized static void resetCeModuleInstanceCount() {
118  correlationModuleInstanceCount = 0; //called when a case is opened in case for some reason counter was not reset
119  }
120 
127  public synchronized static int getCeModuleInstanceCount() {
129  }
130 
136  public synchronized static boolean isFlagNotableItems() {
137  return flagNotableItems;
138  }
139 
145  public synchronized static boolean isFlagSeenDevices() {
146  return flagSeenDevices;
147  }
148 
154  public synchronized static boolean shouldCreateCrProperties() {
155  return createCrProperties;
156  }
157 
163  public synchronized static void setFlagNotableItems(boolean value) {
164  flagNotableItems = value;
165  }
166 
172  public synchronized static void setFlagSeenDevices(boolean value) {
173  flagSeenDevices = value;
174  }
175 
181  public synchronized static void setCreateCrProperties(boolean value) {
182  createCrProperties = value;
183  }
184 
185  @NbBundle.Messages({"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
186  "IngestEventsListener.prevCaseComment.text=Previous Case: ",
187  "IngestEventsListener.ingestmodule.name=Correlation Engine"})
188  static private void postCorrelatedBadArtifactToBlackboard(BlackboardArtifact bbArtifact, List<String> caseDisplayNames) {
189 
190  try {
191  String MODULE_NAME = Bundle.IngestEventsListener_ingestmodule_name();
192 
193  Collection<BlackboardAttribute> attributes = new ArrayList<>();
194  attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME,
195  Bundle.IngestEventsListener_prevTaggedSet_text()));
196  attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME,
197  Bundle.IngestEventsListener_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(",", "", ""))));
198  attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, bbArtifact.getArtifactID()));
199 
200  SleuthkitCase tskCase = bbArtifact.getSleuthkitCase();
201  AbstractFile abstractFile = tskCase.getAbstractFileById(bbArtifact.getObjectID());
202  org.sleuthkit.datamodel.Blackboard tskBlackboard = tskCase.getBlackboard();
203  // Create artifact if it doesn't already exist.
204  if (!tskBlackboard.artifactExists(abstractFile, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT, attributes)) {
205  BlackboardArtifact tifArtifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT);
206  tifArtifact.addAttributes(attributes);
207 
208  try {
209  // index the artifact for keyword search
211  blackboard.indexArtifact(tifArtifact);
213  LOGGER.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS
214  }
215 
216  // fire event to notify UI of this new artifact
217  IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT));
218  }
219  } catch (TskCoreException ex) {
220  LOGGER.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS
221  } catch (IllegalStateException ex) {
222  LOGGER.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
223  }
224  }
225 
232  @NbBundle.Messages({"IngestEventsListener.prevExists.text=Previously Seen Devices (Central Repository)",
233  "# {0} - typeName",
234  "# {1} - count",
235  "IngestEventsListener.prevCount.text=Number of previous {0}: {1}"})
236  static private void postCorrelatedPreviousArtifactToBlackboard(BlackboardArtifact bbArtifact) {
237 
238  try {
239  String MODULE_NAME = Bundle.IngestEventsListener_ingestmodule_name();
240 
241  Collection<BlackboardAttribute> attributes = new ArrayList<>();
242  BlackboardAttribute att = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME,
243  Bundle.IngestEventsListener_prevExists_text());
244  attributes.add(att);
245  attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, bbArtifact.getArtifactID()));
246 
247  SleuthkitCase tskCase = bbArtifact.getSleuthkitCase();
248  AbstractFile abstractFile = bbArtifact.getSleuthkitCase().getAbstractFileById(bbArtifact.getObjectID());
249  org.sleuthkit.datamodel.Blackboard tskBlackboard = tskCase.getBlackboard();
250  // Create artifact if it doesn't already exist.
251  if (!tskBlackboard.artifactExists(abstractFile, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT, attributes)) {
252  BlackboardArtifact tifArtifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT);
253  tifArtifact.addAttributes(attributes);
254 
255  try {
256  // index the artifact for keyword search
258  blackboard.indexArtifact(tifArtifact);
260  LOGGER.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS
261  }
262 
263  // fire event to notify UI of this new artifact
264  IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT));
265  }
266  } catch (TskCoreException ex) {
267  LOGGER.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS
268  } catch (IllegalStateException ex) {
269  LOGGER.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
270  }
271  }
272 
273  private class IngestModuleEventListener implements PropertyChangeListener {
274 
275  @Override
276  public void propertyChange(PropertyChangeEvent evt) {
277  //if ingest is running we want there to check if there is a Correlation Engine module running
278  //sometimes artifacts are generated by DSPs or other sources while ingest is not running
279  //in these cases we still want to create correlation attributes for those artifacts when appropriate
281  EamDb dbManager;
282  try {
283  dbManager = EamDb.getInstance();
284  } catch (EamDbException ex) {
285  LOGGER.log(Level.SEVERE, "Failed to connect to Central Repository database.", ex);
286  return;
287  }
288  switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) {
289  case DATA_ADDED: {
290  //if ingest isn't running create the interesting items otherwise use the ingest module setting to determine if we create interesting items
291  boolean flagNotable = !IngestManager.getInstance().isIngestRunning() || isFlagNotableItems();
292  boolean flagPrevious = !IngestManager.getInstance().isIngestRunning() || isFlagSeenDevices();
293  boolean createAttributes = !IngestManager.getInstance().isIngestRunning() || shouldCreateCrProperties();
294  jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable, flagPrevious, createAttributes));
295  break;
296  }
297  }
298  }
299  }
300  }
301 
302  private class IngestJobEventListener implements PropertyChangeListener {
303 
304  @Override
305  public void propertyChange(PropertyChangeEvent evt) {
306  switch (IngestManager.IngestJobEvent.valueOf(evt.getPropertyName())) {
307  case DATA_SOURCE_ANALYSIS_COMPLETED: {
308  jobProcessingExecutor.submit(new AnalysisCompleteTask());
309  break;
310  }
311  }
312  }
313 
314  }
315 
316  private final class AnalysisCompleteTask implements Runnable {
317 
318  @Override
319  public void run() {
320  // clear the tracker to reduce memory usage
321  if (getCeModuleInstanceCount() == 0) {
322  recentlyAddedCeArtifacts.clear();
323  }
324  //else another instance of the Correlation Engine Module is still being run.
325  } // DATA_SOURCE_ANALYSIS_COMPLETED
326  }
327 
328  private final class DataAddedTask implements Runnable {
329 
330  private final EamDb dbManager;
331  private final PropertyChangeEvent event;
332  private final boolean flagNotableItemsEnabled;
333  private final boolean flagPreviousItemsEnabled;
334  private final boolean createCorrelationAttributes;
335 
336  private DataAddedTask(EamDb db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled, boolean createCorrelationAttributes) {
337  dbManager = db;
338  event = evt;
339  this.flagNotableItemsEnabled = flagNotableItemsEnabled;
340  this.flagPreviousItemsEnabled = flagPreviousItemsEnabled;
341  this.createCorrelationAttributes = createCorrelationAttributes;
342  }
343 
344  @Override
345  public void run() {
346  if (!EamDb.isEnabled()) {
347  return;
348  }
349  final ModuleDataEvent mde = (ModuleDataEvent) event.getOldValue();
350  Collection<BlackboardArtifact> bbArtifacts = mde.getArtifacts();
351  if (null == bbArtifacts) { //the ModuleDataEvents don't always have a collection of artifacts set
352  return;
353  }
354  List<CorrelationAttributeInstance> eamArtifacts = new ArrayList<>();
355 
356  for (BlackboardArtifact bbArtifact : bbArtifacts) {
357  // eamArtifact will be null OR a EamArtifact containing one EamArtifactInstance.
358  List<CorrelationAttributeInstance> convertedArtifacts = EamArtifactUtil.makeInstancesFromBlackboardArtifact(bbArtifact, true);
359  for (CorrelationAttributeInstance eamArtifact : convertedArtifacts) {
360  try {
361  // Only do something with this artifact if it's unique within the job
362  if (recentlyAddedCeArtifacts.add(eamArtifact.toString())) {
363  // Was it previously marked as bad?
364  // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad".
365  // if getKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case,
366  // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB.
367  if (flagNotableItemsEnabled) {
368  List<String> caseDisplayNames;
369  try {
370  caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
371  if (!caseDisplayNames.isEmpty()) {
373  caseDisplayNames);
374  }
376  LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex);
377  }
378  }
379  if (flagPreviousItemsEnabled
380  && (eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.USBID_TYPE_ID
381  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.ICCID_TYPE_ID
382  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.IMEI_TYPE_ID
383  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.IMSI_TYPE_ID
384  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.MAC_TYPE_ID)) {
385  try {
386  Long countPreviousOccurences = dbManager.getCountArtifactInstancesByTypeValue(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
387  if (countPreviousOccurences > 0) {
389  }
391  LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex);
392  }
393  }
394  if (createCorrelationAttributes) {
395  eamArtifacts.add(eamArtifact);
396  }
397  }
398  } catch (EamDbException ex) {
399  LOGGER.log(Level.SEVERE, "Error counting notable artifacts.", ex);
400  }
401  }
402  }
403  if (FALSE == eamArtifacts.isEmpty()) {
404  for (CorrelationAttributeInstance eamArtifact : eamArtifacts) {
405  try {
406  dbManager.addArtifactInstance(eamArtifact);
407  } catch (EamDbException ex) {
408  LOGGER.log(Level.SEVERE, "Error adding artifact to database.", ex); //NON-NLS
409  }
410  }
411  } // DATA_ADDED
412  }
413  }
414 }
Collection< BlackboardArtifact > getArtifacts()
void removeIngestModuleEventListener(final PropertyChangeListener listener)
static List< CorrelationAttributeInstance > makeInstancesFromBlackboardArtifact(BlackboardArtifact artifact, boolean checkEnabled)
static synchronized IngestManager getInstance()
DataAddedTask(EamDb db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled, boolean createCorrelationAttributes)
List< String > getListCasesHavingArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value)
Long getCountArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value)
void removeIngestJobEventListener(final PropertyChangeListener listener)
static void shutDownTaskExecutor(ExecutorService executor)
void addIngestJobEventListener(final PropertyChangeListener listener)
void fireModuleDataEvent(ModuleDataEvent moduleDataEvent)
void addIngestModuleEventListener(final PropertyChangeListener listener)
synchronized void indexArtifact(BlackboardArtifact artifact)
Definition: Blackboard.java:58
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static void postCorrelatedBadArtifactToBlackboard(BlackboardArtifact bbArtifact, List< String > caseDisplayNames)
void addArtifactInstance(CorrelationAttributeInstance eamArtifact)
static synchronized IngestServices getInstance()

Copyright © 2012-2018 Basis Technology. Generated on: Tue Dec 18 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.