Autopsy  4.5.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
HashDbIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011 - 2013 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.hashdatabase;
20 
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.concurrent.atomic.AtomicLong;
28 import java.util.logging.Level;
29 import org.openide.util.NbBundle;
30 import org.openide.util.NbBundle.Messages;
41 import org.sleuthkit.datamodel.AbstractFile;
42 import org.sleuthkit.datamodel.BlackboardArtifact;
43 import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
44 import org.sleuthkit.datamodel.BlackboardAttribute;
45 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
46 import org.sleuthkit.datamodel.HashHitInfo;
47 import org.sleuthkit.datamodel.HashUtility;
48 import org.sleuthkit.datamodel.SleuthkitCase;
49 import org.sleuthkit.datamodel.TskCoreException;
50 import org.sleuthkit.datamodel.TskData;
51 import org.sleuthkit.datamodel.TskException;
52 
53 @NbBundle.Messages({
54  "HashDbIngestModule.noKnownBadHashDbSetMsg=No notable hash set.",
55  "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn=Notable file search will not be executed.",
56  "HashDbIngestModule.noKnownHashDbSetMsg=No known hash set.",
57  "HashDbIngestModule.knownFileSearchWillNotExecuteWarn=Known file search will not be executed."
58 })
59 public class HashDbIngestModule implements FileIngestModule {
60 
61  private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName());
62  private static final int MAX_COMMENT_SIZE = 500;
63  private final IngestServices services = IngestServices.getInstance();
64  private final SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase();
65  private final HashDbManager hashDbManager = HashDbManager.getInstance();
66  private final HashLookupModuleSettings settings;
67  private List<HashDb> knownBadHashSets = new ArrayList<>();
68  private List<HashDb> knownHashSets = new ArrayList<>();
69  private long jobId;
70  private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs = new HashMap<>();
71  private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
73 
74  private static class IngestJobTotals {
75 
76  private AtomicLong totalKnownBadCount = new AtomicLong(0);
77  private AtomicLong totalCalctime = new AtomicLong(0);
78  private AtomicLong totalLookuptime = new AtomicLong(0);
79  }
80 
81  private static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId) {
82  IngestJobTotals totals = totalsForIngestJobs.get(ingestJobId);
83  if (totals == null) {
84  totals = new HashDbIngestModule.IngestJobTotals();
85  totalsForIngestJobs.put(ingestJobId, totals);
86  }
87  return totals;
88  }
89 
90  HashDbIngestModule(HashLookupModuleSettings settings) {
91  this.settings = settings;
92  }
93 
94  @Override
96  jobId = context.getJobId();
97  if (!hashDbManager.verifyAllDatabasesLoadedCorrectly()) {
98  throw new IngestModuleException("Could not load all hash sets");
99  }
100  updateEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets);
101  updateEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets);
102 
103  if (refCounter.incrementAndGet(jobId) == 1) {
104  // initialize job totals
105  getTotalsForIngestJobs(jobId);
106 
107  // if first module for this job then post error msgs if needed
108  if (knownBadHashSets.isEmpty()) {
110  HashLookupModuleFactory.getModuleName(),
111  Bundle.HashDbIngestModule_noKnownBadHashDbSetMsg(),
112  Bundle.HashDbIngestModule_knownBadFileSearchWillNotExecuteWarn()));
113  }
114 
115  if (knownHashSets.isEmpty()) {
117  HashLookupModuleFactory.getModuleName(),
118  Bundle.HashDbIngestModule_noKnownHashDbSetMsg(),
119  Bundle.HashDbIngestModule_knownFileSearchWillNotExecuteWarn()));
120  }
121  }
122  }
123 
130  private void updateEnabledHashSets(List<HashDb> allHashSets, List<HashDb> enabledHashSets) {
131  enabledHashSets.clear();
132  for (HashDb db : allHashSets) {
133  if (settings.isHashSetEnabled(db)) {
134  try {
135  if (db.isValid()) {
136  enabledHashSets.add(db);
137  }
138  } catch (TskCoreException ex) {
139  logger.log(Level.WARNING, "Error getting index status for " + db.getDisplayName()+ " hash set", ex); //NON-NLS
140  }
141  }
142  }
143  }
144 
145  @Override
146  public ProcessResult process(AbstractFile file) {
147  blackboard = Case.getCurrentCase().getServices().getBlackboard();
148 
149  // Skip unallocated space files.
150  if ((file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) ||
151  file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
152  return ProcessResult.OK;
153  }
154 
155  /*
156  * Skip directories. One reason for this is because we won't accurately
157  * calculate hashes of NTFS directories that have content that spans the
158  * IDX_ROOT and IDX_ALLOC artifacts. So we disable that until a solution
159  * for it is developed.
160  */
161  if (file.isDir()) {
162  return ProcessResult.OK;
163  }
164 
165  // bail out if we have no hashes set
166  if ((knownHashSets.isEmpty()) && (knownBadHashSets.isEmpty()) && (!settings.shouldCalculateHashes())) {
167  return ProcessResult.OK;
168  }
169 
170  // Safely get a reference to the totalsForIngestJobs object
171  IngestJobTotals totals = getTotalsForIngestJobs(jobId);
172 
173  // calc hash value
174  String name = file.getName();
175  String md5Hash = file.getMd5Hash();
176  if (md5Hash == null || md5Hash.isEmpty()) {
177  try {
178  long calcstart = System.currentTimeMillis();
179  md5Hash = HashUtility.calculateMd5Hash(file);
180  file.setMd5Hash(md5Hash);
181  long delta = (System.currentTimeMillis() - calcstart);
182  totals.totalCalctime.addAndGet(delta);
183 
184  } catch (IOException ex) {
185  logger.log(Level.WARNING, "Error calculating hash of file " + name, ex); //NON-NLS
187  HashLookupModuleFactory.getModuleName(),
188  NbBundle.getMessage(this.getClass(),
189  "HashDbIngestModule.fileReadErrorMsg",
190  name),
191  NbBundle.getMessage(this.getClass(),
192  "HashDbIngestModule.calcHashValueErr",
193  name)));
194  return ProcessResult.ERROR;
195  }
196  }
197 
198  // look up in notable first
199  boolean foundBad = false;
201  for (HashDb db : knownBadHashSets) {
202  try {
203  long lookupstart = System.currentTimeMillis();
204  HashHitInfo hashInfo = db.lookupMD5(file);
205  if (null != hashInfo) {
206  foundBad = true;
207  totals.totalKnownBadCount.incrementAndGet();
208 
209  file.setKnown(TskData.FileKnown.BAD);
210 
211  String hashSetName = db.getDisplayName();
212 
213  String comment = "";
214  ArrayList<String> comments = hashInfo.getComments();
215  int i = 0;
216  for (String c : comments) {
217  if (++i > 1) {
218  comment += " ";
219  }
220  comment += c;
221  if (comment.length() > MAX_COMMENT_SIZE) {
222  comment = comment.substring(0, MAX_COMMENT_SIZE) + "...";
223  break;
224  }
225  }
226 
227  postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
228  }
229  long delta = (System.currentTimeMillis() - lookupstart);
230  totals.totalLookuptime.addAndGet(delta);
231 
232  } catch (TskException ex) {
233  logger.log(Level.WARNING, "Couldn't lookup notable hash for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
235  HashLookupModuleFactory.getModuleName(),
236  NbBundle.getMessage(this.getClass(),
237  "HashDbIngestModule.hashLookupErrorMsg",
238  name),
239  NbBundle.getMessage(this.getClass(),
240  "HashDbIngestModule.lookingUpKnownBadHashValueErr",
241  name)));
242  ret = ProcessResult.ERROR;
243  }
244  }
245 
246  // If the file is not in the notable sets, search for it in the known sets.
247  // Any hit is sufficient to classify it as known, and there is no need to create
248  // a hit artifact or send a message to the application inbox.
249  if (!foundBad) {
250  for (HashDb db : knownHashSets) {
251  try {
252  long lookupstart = System.currentTimeMillis();
253  if (db.lookupMD5Quick(file)) {
254  file.setKnown(TskData.FileKnown.KNOWN);
255  break;
256  }
257  long delta = (System.currentTimeMillis() - lookupstart);
258  totals.totalLookuptime.addAndGet(delta);
259 
260  } catch (TskException ex) {
261  logger.log(Level.WARNING, "Couldn't lookup known hash for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
263  HashLookupModuleFactory.getModuleName(),
264  NbBundle.getMessage(this.getClass(),
265  "HashDbIngestModule.hashLookupErrorMsg",
266  name),
267  NbBundle.getMessage(this.getClass(),
268  "HashDbIngestModule.lookingUpKnownHashValueErr",
269  name)));
270  ret = ProcessResult.ERROR;
271  }
272  }
273  }
274 
275  return ret;
276  }
277 
278  @Messages({"HashDbIngestModule.indexError.message=Failed to index hashset hit artifact for keyword search."})
279  private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) {
280  try {
281  String MODULE_NAME = NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.moduleName");
282 
283  BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT);
284  Collection<BlackboardAttribute> attributes = new ArrayList<>();
285  //TODO Revisit usage of deprecated constructor as per TSK-583
286  //BlackboardAttribute att2 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), MODULE_NAME, "Known Bad", hashSetName);
287  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, hashSetName));
288  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, MODULE_NAME, md5Hash));
289  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, comment));
290 
291  badFile.addAttributes(attributes);
292 
293  try {
294  // index the artifact for keyword search
295  blackboard.indexArtifact(badFile);
296  } catch (Blackboard.BlackboardException ex) {
297  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + badFile.getArtifactID(), ex); //NON-NLS
299  Bundle.HashDbIngestModule_indexError_message(), badFile.getDisplayName());
300  }
301 
302  if (showInboxMessage) {
303  StringBuilder detailsSb = new StringBuilder();
304  //details
305  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
306  //hit
307  detailsSb.append("<tr>"); //NON-NLS
308  detailsSb.append("<th>") //NON-NLS
309  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.fileName"))
310  .append("</th>"); //NON-NLS
311  detailsSb.append("<td>") //NON-NLS
312  .append(abstractFile.getName())
313  .append("</td>"); //NON-NLS
314  detailsSb.append("</tr>"); //NON-NLS
315 
316  detailsSb.append("<tr>"); //NON-NLS
317  detailsSb.append("<th>") //NON-NLS
318  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.md5Hash"))
319  .append("</th>"); //NON-NLS
320  detailsSb.append("<td>").append(md5Hash).append("</td>"); //NON-NLS
321  detailsSb.append("</tr>"); //NON-NLS
322 
323  detailsSb.append("<tr>"); //NON-NLS
324  detailsSb.append("<th>") //NON-NLS
325  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.hashsetName"))
326  .append("</th>"); //NON-NLS
327  detailsSb.append("<td>").append(hashSetName).append("</td>"); //NON-NLS
328  detailsSb.append("</tr>"); //NON-NLS
329 
330  detailsSb.append("</table>"); //NON-NLS
331 
333  NbBundle.getMessage(this.getClass(),
334  "HashDbIngestModule.postToBB.knownBadMsg",
335  abstractFile.getName()),
336  detailsSb.toString(),
337  abstractFile.getName() + md5Hash,
338  badFile));
339  }
340  services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_HASHSET_HIT, Collections.singletonList(badFile)));
341  } catch (TskException ex) {
342  logger.log(Level.WARNING, "Error creating blackboard artifact", ex); //NON-NLS
343  }
344  }
345 
346  private static synchronized void postSummary(long jobId,
347  List<HashDb> knownBadHashSets, List<HashDb> knownHashSets) {
348  IngestJobTotals jobTotals = getTotalsForIngestJobs(jobId);
349  totalsForIngestJobs.remove(jobId);
350 
351  if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) {
352  StringBuilder detailsSb = new StringBuilder();
353  //details
354  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
355 
356  detailsSb.append("<tr><td>") //NON-NLS
357  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.knownBadsFound"))
358  .append("</td>"); //NON-NLS
359  detailsSb.append("<td>").append(jobTotals.totalKnownBadCount.get()).append("</td></tr>"); //NON-NLS
360 
361  detailsSb.append("<tr><td>") //NON-NLS
362  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.totalCalcTime"))
363  .append("</td><td>").append(jobTotals.totalCalctime.get()).append("</td></tr>\n"); //NON-NLS
364  detailsSb.append("<tr><td>") //NON-NLS
365  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.totalLookupTime"))
366  .append("</td><td>").append(jobTotals.totalLookuptime.get()).append("</td></tr>\n"); //NON-NLS
367  detailsSb.append("</table>"); //NON-NLS
368 
369  detailsSb.append("<p>") //NON-NLS
370  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.databasesUsed"))
371  .append("</p>\n<ul>"); //NON-NLS
372  for (HashDb db : knownBadHashSets) {
373  detailsSb.append("<li>").append(db.getHashSetName()).append("</li>\n"); //NON-NLS
374  }
375 
376  detailsSb.append("</ul>"); //NON-NLS
377 
380  HashLookupModuleFactory.getModuleName(),
381  NbBundle.getMessage(HashDbIngestModule.class,
382  "HashDbIngestModule.complete.hashLookupResults"),
383  detailsSb.toString()));
384  }
385  }
386 
387  @Override
388  public void shutDown() {
389  if (refCounter.decrementAndGet(jobId) == 0) {
390  postSummary(jobId, knownBadHashSets, knownHashSets);
391  }
392  }
393 }
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context)
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId)
void updateEnabledHashSets(List< HashDb > allHashSets, List< HashDb > enabledHashSets)
void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage)
void postMessage(final IngestMessage message)
void fireModuleDataEvent(ModuleDataEvent moduleDataEvent)
static synchronized void postSummary(long jobId, List< HashDb > knownBadHashSets, List< HashDb > knownHashSets)
static void error(String title, String message)
synchronized void indexArtifact(BlackboardArtifact artifact)
Definition: Blackboard.java:59
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static IngestMessage createWarningMessage(String source, String subject, String detailsHtml)
static synchronized IngestServices getInstance()

Copyright © 2012-2016 Basis Technology. Generated on: Tue Feb 20 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.