Autopsy  4.13.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
CentralRepoIngestModule.java
Go to the documentation of this file.
1 /*
2  * Central Repository
3  *
4  * Copyright 2011-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.centralrepository.ingestmodule;
20 
21 import java.util.Arrays;
22 import java.util.Collection;
23 import java.util.List;
24 import java.util.logging.Level;
25 import java.util.stream.Collectors;
26 import org.openide.util.NbBundle.Messages;
48 import org.sleuthkit.datamodel.AbstractFile;
49 import org.sleuthkit.datamodel.Blackboard;
50 import org.sleuthkit.datamodel.BlackboardArtifact;
51 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT;
52 import org.sleuthkit.datamodel.BlackboardAttribute;
53 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT;
54 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME;
55 import org.sleuthkit.datamodel.HashUtility;
56 import org.sleuthkit.datamodel.TskCoreException;
57 import org.sleuthkit.datamodel.TskData;
58 
63 @Messages({"CentralRepoIngestModule.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
64  "CentralRepoIngestModule.prevCaseComment.text=Previous Case: "})
65 final class CentralRepoIngestModule implements FileIngestModule {
66 
67  private static final String MODULE_NAME = CentralRepoIngestModuleFactory.getModuleName();
68 
69  static final boolean DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS = true;
70  static final boolean DEFAULT_FLAG_PREVIOUS_DEVICES = true;
71  static final boolean DEFAULT_CREATE_CR_PROPERTIES = true;
72 
73  private final static Logger logger = Logger.getLogger(CentralRepoIngestModule.class.getName());
74  private final IngestServices services = IngestServices.getInstance();
75  private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
76  private static final IngestModuleReferenceCounter warningMsgRefCounter = new IngestModuleReferenceCounter();
77  private long jobId;
78  private CorrelationCase eamCase;
79  private CorrelationDataSource eamDataSource;
80  private CorrelationAttributeInstance.Type filesType;
81  private final boolean flagTaggedNotableItems;
82  private final boolean flagPreviouslySeenDevices;
83  private Blackboard blackboard;
84  private final boolean createCorrelationProperties;
85 
91  CentralRepoIngestModule(IngestSettings settings) {
92  flagTaggedNotableItems = settings.isFlagTaggedNotableItems();
93  flagPreviouslySeenDevices = settings.isFlagPreviousDevices();
94  createCorrelationProperties = settings.shouldCreateCorrelationProperties();
95  }
96 
97  @Override
98  public ProcessResult process(AbstractFile abstractFile) {
99  if (EamDb.isEnabled() == false) {
100  /*
101  * Not signaling an error for now. This is a workaround for the way
102  * all newly didscovered ingest modules are automatically anabled.
103  *
104  * TODO (JIRA-2731): Add isEnabled API for ingest modules.
105  */
106  return ProcessResult.OK;
107  }
108 
109  try {
110  blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard();
111  } catch (NoCurrentCaseException ex) {
112  logger.log(Level.SEVERE, "Exception while getting open case.", ex);
113  return ProcessResult.ERROR;
114  }
115 
116  if (!EamArtifactUtil.isSupportedAbstractFileType(abstractFile)) {
117  return ProcessResult.OK;
118  }
119 
120  if (abstractFile.getKnown() == TskData.FileKnown.KNOWN) {
121  return ProcessResult.OK;
122  }
123 
124  EamDb dbManager;
125  try {
126  dbManager = EamDb.getInstance();
127  } catch (EamDbException ex) {
128  logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
129  return ProcessResult.ERROR;
130  }
131 
132  // only continue if we are correlating filesType
133  if (!filesType.isEnabled()) {
134  return ProcessResult.OK;
135  }
136 
137  // get the hash because we're going to correlate it
138  String md5 = abstractFile.getMd5Hash();
139  if ((md5 == null) || (HashUtility.isNoDataMd5(md5))) {
140  return ProcessResult.OK;
141  }
142 
143  /*
144  * Search the central repo to see if this file was previously marked as
145  * being bad. Create artifact if it was.
146  */
147  if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) {
148  try {
149  TimingMetric timingMetric = HealthMonitor.getTimingMetric("Correlation Engine: Notable artifact query");
150  List<String> caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5);
151  HealthMonitor.submitTimingMetric(timingMetric);
152  if (!caseDisplayNamesList.isEmpty()) {
153  postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList);
154  }
155  } catch (EamDbException ex) {
156  logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS
157  return ProcessResult.ERROR;
159  logger.log(Level.INFO, "Error searching database for artifact.", ex); // NON-NLS
160  return ProcessResult.ERROR;
161  }
162  }
163 
164  // insert this file into the central repository
165  if (createCorrelationProperties) {
166  try {
168  filesType,
169  md5,
170  eamCase,
171  eamDataSource,
172  abstractFile.getParentPath() + abstractFile.getName(),
173  null,
174  TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database.
175  ,
176  abstractFile.getId());
177  dbManager.addAttributeInstanceBulk(cefi);
178  } catch (EamDbException ex) {
179  logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
180  return ProcessResult.ERROR;
182  logger.log(Level.INFO, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
183  return ProcessResult.ERROR;
184  }
185  }
186  return ProcessResult.OK;
187  }
188 
189  @Override
190  public void shutDown() {
192 
193  if ((EamDb.isEnabled() == false) || (eamCase == null) || (eamDataSource == null)) {
194  return;
195  }
196  EamDb dbManager;
197  try {
198  dbManager = EamDb.getInstance();
199  } catch (EamDbException ex) {
200  logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
201  return;
202  }
203  try {
204  dbManager.commitAttributeInstancesBulk();
205  } catch (EamDbException ex) {
206  logger.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS
207  }
208  try {
209  Long count = dbManager.getCountArtifactInstancesByCaseDataSource(eamDataSource);
210  logger.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, eamCase.getDisplayName(), eamDataSource.getName()}); // NON-NLS
211  } catch (EamDbException ex) {
212  logger.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS
213  }
214 
215  // TODO: once we implement shared cache, if refCounter is 1, then submit data in bulk.
216  refCounter.decrementAndGet(jobId);
217  }
218 
219  // see ArtifactManagerTimeTester for details
220  @Messages({
221  "CentralRepoIngestModule.notfyBubble.title=Central Repository Not Initialized",
222  "CentralRepoIngestModule.errorMessage.isNotEnabled=Central repository settings are not initialized, cannot run Correlation Engine ingest module."
223  })
224  @Override
225  public void startUp(IngestJobContext context) throws IngestModuleException {
227 
228  /*
229  * Tell the IngestEventsListener to flag notable items based on the
230  * current module's configuration. This is a work around for the lack of
231  * an artifacts pipeline. Note that this can be changed by another
232  * module instance. All modules are affected by the value. While not
233  * ideal, this will be good enough until a better solution can be
234  * posited.
235  *
236  * Note: Flagging cannot be disabled if any other instances of the
237  * Correlation Engine module are running. This restriction is to prevent
238  * missing results in the case where the first module is flagging
239  * notable items, and the proceeding module (with flagging disabled)
240  * causes the first to stop flagging.
241  */
243  IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems);
244  }
246  IngestEventsListener.setFlagSeenDevices(flagPreviouslySeenDevices);
247  }
249  IngestEventsListener.setCreateCrProperties(createCorrelationProperties);
250  }
251 
252  if (EamDb.isEnabled() == false) {
253  /*
254  * Not throwing the customary exception for now. This is a
255  * workaround for the way all newly didscovered ingest modules are
256  * automatically anabled.
257  *
258  * TODO (JIRA-2731): Add isEnabled API for ingest modules.
259  */
261  if (1L == warningMsgRefCounter.incrementAndGet(jobId)) {
262  MessageNotifyUtil.Notify.warn(Bundle.CentralRepoIngestModule_notfyBubble_title(), Bundle.CentralRepoIngestModule_errorMessage_isNotEnabled());
263  }
264  }
265  return;
266  }
267  Case autopsyCase;
268  try {
269  autopsyCase = Case.getCurrentCaseThrows();
270  } catch (NoCurrentCaseException ex) {
271  logger.log(Level.SEVERE, "Exception while getting open case.", ex);
272  throw new IngestModuleException("Exception while getting open case.", ex);
273  }
274 
275  // Don't allow sqlite central repo databases to be used for multi user cases
276  if ((autopsyCase.getCaseType() == Case.CaseType.MULTI_USER_CASE)
278  logger.log(Level.SEVERE, "Cannot run correlation engine on a multi-user case with a SQLite central repository.");
279  throw new IngestModuleException("Cannot run on a multi-user case with a SQLite central repository."); // NON-NLS
280  }
281  jobId = context.getJobId();
282 
283  EamDb centralRepoDb;
284  try {
285  centralRepoDb = EamDb.getInstance();
286  } catch (EamDbException ex) {
287  logger.log(Level.SEVERE, "Error connecting to central repository database.", ex); // NON-NLS
288  throw new IngestModuleException("Error connecting to central repository database.", ex); // NON-NLS
289  }
290 
291  try {
293  } catch (EamDbException ex) {
294  logger.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
295  throw new IngestModuleException("Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
296  }
297 
298  try {
299  eamCase = centralRepoDb.getCase(autopsyCase);
300  } catch (EamDbException ex) {
301  throw new IngestModuleException("Unable to get case from central repository database ", ex);
302  }
303 
304  try {
305  eamDataSource = CorrelationDataSource.fromTSKDataSource(eamCase, context.getDataSource());
306  } catch (EamDbException ex) {
307  logger.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS
308  throw new IngestModuleException("Error getting data source info.", ex); // NON-NLS
309  }
310  // TODO: once we implement a shared cache, load/init it here w/ syncronized and define reference counter
311  // if we are the first thread / module for this job, then make sure the case
312  // and image exist in the DB before we associate artifacts with it.
313  if (refCounter.incrementAndGet(jobId)
314  == 1) {
315  // ensure we have this data source in the EAM DB
316  try {
317  if (null == centralRepoDb.getDataSource(eamCase, eamDataSource.getDataSourceObjectID())) {
318  centralRepoDb.newDataSource(eamDataSource);
319  }
320  } catch (EamDbException ex) {
321  logger.log(Level.SEVERE, "Error adding data source to Central Repository.", ex); // NON-NLS
322  throw new IngestModuleException("Error adding data source to Central Repository.", ex); // NON-NLS
323  }
324 
325  }
326  }
327 
334  private void postCorrelatedBadFileToBlackboard(AbstractFile abstractFile, List<String> caseDisplayNames) {
335 
336  Collection<BlackboardAttribute> attributes = Arrays.asList(
337  new BlackboardAttribute(
338  TSK_SET_NAME, MODULE_NAME,
339  Bundle.CentralRepoIngestModule_prevTaggedSet_text()),
340  new BlackboardAttribute(
341  TSK_COMMENT, MODULE_NAME,
342  Bundle.CentralRepoIngestModule_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(","))));
343  try {
344 
345  // Create artifact if it doesn't already exist.
346  if (!blackboard.artifactExists(abstractFile, TSK_INTERESTING_FILE_HIT, attributes)) {
347  BlackboardArtifact tifArtifact = abstractFile.newArtifact(TSK_INTERESTING_FILE_HIT);
348  tifArtifact.addAttributes(attributes);
349  try {
350  // index the artifact for keyword search
351  blackboard.postArtifact(tifArtifact, MODULE_NAME);
352  } catch (Blackboard.BlackboardException ex) {
353  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS
354  }
355  // send inbox message
356  sendBadFileInboxMessage(tifArtifact, abstractFile.getName(), abstractFile.getMd5Hash(), caseDisplayNames);
357  }
358  } catch (TskCoreException ex) {
359  logger.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS
360  } catch (IllegalStateException ex) {
361  logger.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
362  }
363  }
364 
365  @Messages({
366  "CentralRepoIngestModule_notable_message_header=<html>A file in this data source was previously seen and tagged as Notable.<br>",
367  "CentralRepoIngestModel_name_header=Name:<br>",
368  "CentralRepoIngestModel_previous_case_header=<br>Previous Cases:<br>",
369  "# {0} - Name of file that is Notable",
370  "CentralRepoIngestModule_postToBB_knownBadMsg=Notable: {0}"
371  })
372 
382  private void sendBadFileInboxMessage(BlackboardArtifact artifact, String name, String md5Hash, List<String> caseDisplayNames) {
383  StringBuilder detailsSb = new StringBuilder(1024);
384 
385  detailsSb.append(Bundle.CentralRepoIngestModule_notable_message_header()).append(Bundle.CentralRepoIngestModel_name_header());
386  detailsSb.append(name).append(Bundle.CentralRepoIngestModel_previous_case_header());
387  for (String str : caseDisplayNames) {
388  detailsSb.append(str).append("<br>");
389  }
390  detailsSb.append("</html>");
392  Bundle.CentralRepoIngestModule_postToBB_knownBadMsg(name),
393  detailsSb.toString(),
394  name + md5Hash,
395  artifact));
396  }
397 }
CorrelationDataSource newDataSource(CorrelationDataSource eamDataSource)
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
List< String > getListCasesHavingArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value)
static CorrelationDataSource fromTSKDataSource(CorrelationCase correlationCase, Content dataSource)
static TimingMetric getTimingMetric(String name)
void addAttributeInstanceBulk(CorrelationAttributeInstance eamArtifact)
CorrelationDataSource getDataSource(CorrelationCase correlationCase, Long caseDbDataSourceId)
CorrelationAttributeInstance.Type getCorrelationTypeById(int typeId)
void postMessage(final IngestMessage message)
Long getCountArtifactInstancesByCaseDataSource(CorrelationDataSource correlationDataSource)
static void submitTimingMetric(TimingMetric metric)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static void warn(String title, String message)
static synchronized IngestServices getInstance()

Copyright © 2012-2019 Basis Technology. Generated on: Tue Jan 7 2020
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.