Autopsy  4.19.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
CentralRepoIngestModule.java
Go to the documentation of this file.
1 /*
2  * Central Repository
3  *
4  * Copyright 2011-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.centralrepository.ingestmodule;
20 
21 import java.util.Arrays;
22 import java.util.Collection;
23 import java.util.List;
24 import java.util.logging.Level;
25 import java.util.stream.Collectors;
26 import org.openide.util.NbBundle.Messages;
48 import org.sleuthkit.datamodel.AbstractFile;
49 import org.sleuthkit.datamodel.Blackboard;
50 import org.sleuthkit.datamodel.BlackboardArtifact;
51 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT;
52 import org.sleuthkit.datamodel.BlackboardAttribute;
53 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT;
54 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME;
55 import org.sleuthkit.datamodel.HashUtility;
56 import org.sleuthkit.datamodel.TskCoreException;
57 import org.sleuthkit.datamodel.TskData;
59 import org.sleuthkit.datamodel.Score;
60 
65 @Messages({"CentralRepoIngestModule.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
66  "CentralRepoIngestModule.prevCaseComment.text=Previous Case: "})
67 final class CentralRepoIngestModule implements FileIngestModule {
68 
69  private static final String MODULE_NAME = CentralRepoIngestModuleFactory.getModuleName();
70  static final boolean DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS = false;
71  static final boolean DEFAULT_FLAG_PREVIOUS_DEVICES = false;
72  static final boolean DEFAULT_CREATE_CR_PROPERTIES = true;
73 
74  private final static Logger logger = Logger.getLogger(CentralRepoIngestModule.class.getName());
75  private final IngestServices services = IngestServices.getInstance();
76  private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
77  private static final IngestModuleReferenceCounter warningMsgRefCounter = new IngestModuleReferenceCounter();
78  private long jobId;
79  private CorrelationCase eamCase;
80  private CorrelationDataSource eamDataSource;
81  private CorrelationAttributeInstance.Type filesType;
82  private final boolean flagTaggedNotableItems;
83  private final boolean flagPreviouslySeenDevices;
84  private Blackboard blackboard;
85  private final boolean createCorrelationProperties;
86 
92  CentralRepoIngestModule(IngestSettings settings) {
93  flagTaggedNotableItems = settings.isFlagTaggedNotableItems();
94  flagPreviouslySeenDevices = settings.isFlagPreviousDevices();
95  createCorrelationProperties = settings.shouldCreateCorrelationProperties();
96  }
97 
98  @Override
99  public ProcessResult process(AbstractFile abstractFile) {
100  if (CentralRepository.isEnabled() == false) {
101  /*
102  * Not signaling an error for now. This is a workaround for the way
103  * all newly didscovered ingest modules are automatically anabled.
104  *
105  * TODO (JIRA-2731): Add isEnabled API for ingest modules.
106  */
107  return ProcessResult.OK;
108  }
109 
110  try {
111  blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard();
112  } catch (NoCurrentCaseException ex) {
113  logger.log(Level.SEVERE, "Exception while getting open case.", ex);
114  return ProcessResult.ERROR;
115  }
116 
117  if (!CorrelationAttributeUtil.isSupportedAbstractFileType(abstractFile)) {
118  return ProcessResult.OK;
119  }
120 
121  if (abstractFile.getKnown() == TskData.FileKnown.KNOWN) {
122  return ProcessResult.OK;
123  }
124 
125  CentralRepository dbManager;
126  try {
127  dbManager = CentralRepository.getInstance();
128  } catch (CentralRepoException ex) {
129  logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
130  return ProcessResult.ERROR;
131  }
132 
133  // only continue if we are correlating filesType
134  if (!filesType.isEnabled()) {
135  return ProcessResult.OK;
136  }
137 
138  // get the hash because we're going to correlate it
139  String md5 = abstractFile.getMd5Hash();
140  if ((md5 == null) || (HashUtility.isNoDataMd5(md5))) {
141  return ProcessResult.OK;
142  }
143 
144  /*
145  * Search the central repo to see if this file was previously marked as
146  * being bad. Create artifact if it was.
147  */
148  if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) {
149  try {
150  TimingMetric timingMetric = HealthMonitor.getTimingMetric("Central Repository: Notable artifact query");
151  List<String> caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5);
152  HealthMonitor.submitTimingMetric(timingMetric);
153  if (!caseDisplayNamesList.isEmpty()) {
154  postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList);
155  }
156  } catch (CentralRepoException ex) {
157  logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS
158  return ProcessResult.ERROR;
159  } catch (CorrelationAttributeNormalizationException ex) {
160  logger.log(Level.INFO, "Error searching database for artifact.", ex); // NON-NLS
161  return ProcessResult.ERROR;
162  }
163  }
164 
165  // insert this file into the central repository
166  if (createCorrelationProperties) {
167  try {
168  CorrelationAttributeInstance cefi = new CorrelationAttributeInstance(
169  filesType,
170  md5,
171  eamCase,
172  eamDataSource,
173  abstractFile.getParentPath() + abstractFile.getName(),
174  null,
175  TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database.
176  ,
177  abstractFile.getId());
178  dbManager.addAttributeInstanceBulk(cefi);
179  } catch (CentralRepoException ex) {
180  logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
181  return ProcessResult.ERROR;
182  } catch (CorrelationAttributeNormalizationException ex) {
183  logger.log(Level.INFO, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
184  return ProcessResult.ERROR;
185  }
186  }
187  return ProcessResult.OK;
188  }
189 
190  @Override
191  public void shutDown() {
192  IngestEventsListener.decrementCorrelationEngineModuleCount();
193 
194  if ((CentralRepository.isEnabled() == false) || (eamCase == null) || (eamDataSource == null)) {
195  return;
196  }
197  CentralRepository dbManager;
198  try {
199  dbManager = CentralRepository.getInstance();
200  } catch (CentralRepoException ex) {
201  logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
202  return;
203  }
204  try {
205  dbManager.commitAttributeInstancesBulk();
206  } catch (CentralRepoException ex) {
207  logger.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS
208  }
209  try {
210  Long count = dbManager.getCountArtifactInstancesByCaseDataSource(eamDataSource);
211  logger.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, eamCase.getDisplayName(), eamDataSource.getName()}); // NON-NLS
212  } catch (CentralRepoException ex) {
213  logger.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS
214  }
215 
216  // TODO: once we implement shared cache, if refCounter is 1, then submit data in bulk.
217  refCounter.decrementAndGet(jobId);
218  }
219 
220  // see ArtifactManagerTimeTester for details
221  @Messages({
222  "CentralRepoIngestModule.notfyBubble.title=Central Repository Not Initialized",
223  "CentralRepoIngestModule.errorMessage.isNotEnabled=Central repository settings are not initialized, cannot run Central Repository ingest module."
224  })
225  @Override
226  public void startUp(IngestJobContext context) throws IngestModuleException {
227  IngestEventsListener.incrementCorrelationEngineModuleCount();
228 
229  /*
230  * Tell the IngestEventsListener to flag notable items based on the
231  * current module's configuration. This is a work around for the lack of
232  * an artifacts pipeline. Note that this can be changed by another
233  * module instance. All modules are affected by the value. While not
234  * ideal, this will be good enough until a better solution can be
235  * posited.
236  *
237  * Note: Flagging cannot be disabled if any other instances of the
238  * Central Repository module are running. This restriction is to prevent
239  * missing results in the case where the first module is flagging
240  * notable items, and the proceeding module (with flagging disabled)
241  * causes the first to stop flagging.
242  */
243  if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.isFlagNotableItems()) {
244  IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems);
245  }
246  if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.isFlagSeenDevices()) {
247  IngestEventsListener.setFlagSeenDevices(flagPreviouslySeenDevices);
248  }
249  if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.shouldCreateCrProperties()) {
250  IngestEventsListener.setCreateCrProperties(createCorrelationProperties);
251  }
252 
253  if (CentralRepository.isEnabled() == false) {
254  /*
255  * Not throwing the customary exception for now. This is a
256  * workaround for the way all newly didscovered ingest modules are
257  * automatically anabled.
258  *
259  * TODO (JIRA-2731): Add isEnabled API for ingest modules.
260  */
261  if (RuntimeProperties.runningWithGUI()) {
262  if (1L == warningMsgRefCounter.incrementAndGet(jobId)) {
263  MessageNotifyUtil.Notify.warn(Bundle.CentralRepoIngestModule_notfyBubble_title(), Bundle.CentralRepoIngestModule_errorMessage_isNotEnabled());
264  }
265  }
266  return;
267  }
268  Case autopsyCase;
269  try {
270  autopsyCase = Case.getCurrentCaseThrows();
271  } catch (NoCurrentCaseException ex) {
272  logger.log(Level.SEVERE, "Exception while getting open case.", ex);
273  throw new IngestModuleException("Exception while getting open case.", ex);
274  }
275 
276  // Don't allow sqlite central repo databases to be used for multi user cases
277  if ((autopsyCase.getCaseType() == Case.CaseType.MULTI_USER_CASE)
278  && (CentralRepoDbManager.getSavedDbChoice().getDbPlatform() == CentralRepoPlatforms.SQLITE)) {
279  logger.log(Level.SEVERE, "Cannot run Central Repository ingest module on a multi-user case with a SQLite central repository.");
280  throw new IngestModuleException("Cannot run on a multi-user case with a SQLite central repository."); // NON-NLS
281  }
282  jobId = context.getJobId();
283 
284  CentralRepository centralRepoDb;
285  try {
286  centralRepoDb = CentralRepository.getInstance();
287  } catch (CentralRepoException ex) {
288  logger.log(Level.SEVERE, "Error connecting to central repository database.", ex); // NON-NLS
289  throw new IngestModuleException("Error connecting to central repository database.", ex); // NON-NLS
290  }
291 
292  try {
293  filesType = centralRepoDb.getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID);
294  } catch (CentralRepoException ex) {
295  logger.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
296  throw new IngestModuleException("Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
297  }
298 
299  try {
300  eamCase = centralRepoDb.getCase(autopsyCase);
301  } catch (CentralRepoException ex) {
302  throw new IngestModuleException("Unable to get case from central repository database ", ex);
303  }
304 
305  try {
306  eamDataSource = CorrelationDataSource.fromTSKDataSource(eamCase, context.getDataSource());
307  } catch (CentralRepoException ex) {
308  logger.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS
309  throw new IngestModuleException("Error getting data source info.", ex); // NON-NLS
310  }
311  // TODO: once we implement a shared cache, load/init it here w/ syncronized and define reference counter
312  // if we are the first thread / module for this job, then make sure the case
313  // and image exist in the DB before we associate artifacts with it.
314  if (refCounter.incrementAndGet(jobId)
315  == 1) {
316  // ensure we have this data source in the EAM DB
317  try {
318  if (null == centralRepoDb.getDataSource(eamCase, eamDataSource.getDataSourceObjectID())) {
319  centralRepoDb.newDataSource(eamDataSource);
320  }
321  } catch (CentralRepoException ex) {
322  logger.log(Level.SEVERE, "Error adding data source to Central Repository.", ex); // NON-NLS
323  throw new IngestModuleException("Error adding data source to Central Repository.", ex); // NON-NLS
324  }
325 
326  }
327  }
328 
335  private void postCorrelatedBadFileToBlackboard(AbstractFile abstractFile, List<String> caseDisplayNames) {
336  Collection<BlackboardAttribute> attributes = Arrays.asList(
337  new BlackboardAttribute(
338  TSK_SET_NAME, MODULE_NAME,
339  Bundle.CentralRepoIngestModule_prevTaggedSet_text()),
340  new BlackboardAttribute(
341  TSK_COMMENT, MODULE_NAME,
342  Bundle.CentralRepoIngestModule_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(","))));
343  try {
344 
345  // Create artifact if it doesn't already exist.
346  if (!blackboard.artifactExists(abstractFile, TSK_INTERESTING_FILE_HIT, attributes)) {
347  BlackboardArtifact tifArtifact = abstractFile.newAnalysisResult(
348  BlackboardArtifact.Type.TSK_INTERESTING_FILE_HIT, Score.SCORE_LIKELY_NOTABLE,
349  null, Bundle.CentralRepoIngestModule_prevTaggedSet_text(), null, attributes)
350  .getAnalysisResult();
351  try {
352  // index the artifact for keyword search
353  blackboard.postArtifact(tifArtifact, MODULE_NAME);
354  } catch (Blackboard.BlackboardException ex) {
355  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS
356  }
357  // send inbox message
358  sendBadFileInboxMessage(tifArtifact, abstractFile.getName(), abstractFile.getMd5Hash(), caseDisplayNames);
359  }
360  } catch (TskCoreException ex) {
361  logger.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS
362  } catch (IllegalStateException ex) {
363  logger.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
364  }
365  }
366 
367  @Messages({
368  "CentralRepoIngestModule_notable_message_header=<html>A file in this data source was previously seen and tagged as Notable.<br>",
369  "CentralRepoIngestModel_name_header=Name:<br>",
370  "CentralRepoIngestModel_previous_case_header=<br>Previous Cases:<br>",
371  "# {0} - Name of file that is Notable",
372  "CentralRepoIngestModule_postToBB_knownBadMsg=Notable: {0}"
373  })
374 
384  private void sendBadFileInboxMessage(BlackboardArtifact artifact, String name, String md5Hash, List<String> caseDisplayNames) {
385  StringBuilder detailsSb = new StringBuilder(1024);
386 
387  detailsSb.append(Bundle.CentralRepoIngestModule_notable_message_header()).append(Bundle.CentralRepoIngestModel_name_header());
388  detailsSb.append(name).append(Bundle.CentralRepoIngestModel_previous_case_header());
389  for (String str : caseDisplayNames) {
390  detailsSb.append(str).append("<br>");
391  }
392  detailsSb.append("</html>");
393  services.postMessage(IngestMessage.createDataMessage(CentralRepoIngestModuleFactory.getModuleName(),
394  Bundle.CentralRepoIngestModule_postToBB_knownBadMsg(name),
395  detailsSb.toString(),
396  name + md5Hash,
397  artifact));
398  }
399 }

Copyright © 2012-2021 Basis Technology. Generated on: Fri Aug 6 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.