Autopsy  3.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
HashDbIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011 - 2013 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.hashdatabase;
20 
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.concurrent.atomic.AtomicLong;
27 import java.util.logging.Level;
28 import org.openide.util.NbBundle;
48 
49 public class HashDbIngestModule implements FileIngestModule {
50  private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName());
51  private static final int MAX_COMMENT_SIZE = 500;
55  private final HashLookupModuleSettings settings;
56  private List<HashDb> knownBadHashSets = new ArrayList<>();
57  private List<HashDb> knownHashSets = new ArrayList<>();
58  private long jobId;
59  private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs = new HashMap<>();
61 
62  private static class IngestJobTotals {
63  private AtomicLong totalKnownBadCount = new AtomicLong(0);
64  private AtomicLong totalCalctime = new AtomicLong(0);
65  private AtomicLong totalLookuptime = new AtomicLong(0);
66  }
67 
68  private static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId) {
69  IngestJobTotals totals = totalsForIngestJobs.get(ingestJobId);
70  if (totals == null) {
71  totals = new HashDbIngestModule.IngestJobTotals();
72  totalsForIngestJobs.put(ingestJobId, totals);
73  }
74  return totals;
75  }
76 
77  HashDbIngestModule(HashLookupModuleSettings settings) {
78  this.settings = settings;
79  }
80 
81  @Override
83  jobId = context.getJobId();
86 
87  if (refCounter.incrementAndGet(jobId) == 1) {
88  // if first module for this job then post error msgs if needed
89 
90  if (knownBadHashSets.isEmpty()) {
92  HashLookupModuleFactory.getModuleName(),
93  NbBundle.getMessage(this.getClass(),
94  "HashDbIngestModule.noKnownBadHashDbSetMsg"),
95  NbBundle.getMessage(this.getClass(),
96  "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn")));
97  }
98 
99  if (knownHashSets.isEmpty()) {
101  HashLookupModuleFactory.getModuleName(),
102  NbBundle.getMessage(this.getClass(),
103  "HashDbIngestModule.noKnownHashDbSetMsg"),
104  NbBundle.getMessage(this.getClass(),
105  "HashDbIngestModule.knownFileSearchWillNotExecuteWarn")));
106  }
107  }
108  }
109 
115  private void updateEnabledHashSets(List<HashDb> allHashSets, List<HashDb> enabledHashSets) {
116  enabledHashSets.clear();
117  for (HashDb db : allHashSets) {
118  if (settings.isHashSetEnabled(db.getHashSetName())) {
119  try {
120  if (db.hasIndex()) {
121  enabledHashSets.add(db);
122  }
123  }
124  catch (TskCoreException ex) {
125  logger.log(Level.WARNING, "Error getting index status for " + db.getHashSetName() +" hash database", ex); //NON-NLS
126  }
127  }
128  }
129  }
130 
131  @Override
133  // Skip unallocated space files.
135  return ProcessResult.OK;
136  }
137 
138  /* Skip directories. One reason for this is because we won't accurately
139  * calculate hashes of NTFS directories that have content that spans the
140  * IDX_ROOT and IDX_ALLOC artifacts. So we disable that until a solution for
141  * it is developed.
142  */
143  if (file.isDir()) {
144  return ProcessResult.OK;
145  }
146 
147  // bail out if we have no hashes set
148  if ((knownHashSets.isEmpty()) && (knownBadHashSets.isEmpty()) && (!settings.shouldCalculateHashes())) {
149  return ProcessResult.OK;
150  }
151 
152  // Safely get a reference to the totalsForIngestJobs object
153  IngestJobTotals totals = getTotalsForIngestJobs(jobId);
154 
155  // calc hash value
156  String name = file.getName();
157  String md5Hash = file.getMd5Hash();
158  if (md5Hash == null || md5Hash.isEmpty()) {
159  try {
160  long calcstart = System.currentTimeMillis();
161  md5Hash = HashUtility.calculateMd5(file);
162  long delta = (System.currentTimeMillis() - calcstart);
163  totals.totalCalctime.addAndGet(delta);
164 
165  } catch (IOException ex) {
166  logger.log(Level.WARNING, "Error calculating hash of file " + name, ex); //NON-NLS
168  HashLookupModuleFactory.getModuleName(),
169  NbBundle.getMessage(this.getClass(),
170  "HashDbIngestModule.fileReadErrorMsg",
171  name),
172  NbBundle.getMessage(this.getClass(),
173  "HashDbIngestModule.calcHashValueErr",
174  name)));
175  return ProcessResult.ERROR;
176  }
177  }
178 
179  // look up in known bad first
180  boolean foundBad = false;
182  for (HashDb db : knownBadHashSets) {
183  try {
184  long lookupstart = System.currentTimeMillis();
185  HashHitInfo hashInfo = db.lookupMD5(file);
186  if (null != hashInfo) {
187  foundBad = true;
188  totals.totalKnownBadCount.incrementAndGet();
189 
190  try {
191  skCase.setKnown(file, TskData.FileKnown.BAD);
192  } catch (TskException ex) {
193  logger.log(Level.WARNING, "Couldn't set known bad state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
195  HashLookupModuleFactory.getModuleName(),
196  NbBundle.getMessage(this.getClass(),
197  "HashDbIngestModule.hashLookupErrorMsg",
198  name),
199  NbBundle.getMessage(this.getClass(),
200  "HashDbIngestModule.settingKnownBadStateErr",
201  name)));
202  ret = ProcessResult.ERROR;
203  }
204  String hashSetName = db.getHashSetName();
205 
206  String comment = "";
207  ArrayList<String> comments = hashInfo.getComments();
208  int i = 0;
209  for (String c : comments) {
210  if (++i > 1) {
211  comment += " ";
212  }
213  comment += c;
214  if (comment.length() > MAX_COMMENT_SIZE) {
215  comment = comment.substring(0, MAX_COMMENT_SIZE) + "...";
216  break;
217  }
218  }
219 
220  postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
221  }
222  long delta = (System.currentTimeMillis() - lookupstart);
223  totals.totalLookuptime.addAndGet(delta);
224 
225  } catch (TskException ex) {
226  logger.log(Level.WARNING, "Couldn't lookup known bad hash for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
228  HashLookupModuleFactory.getModuleName(),
229  NbBundle.getMessage(this.getClass(),
230  "HashDbIngestModule.hashLookupErrorMsg",
231  name),
232  NbBundle.getMessage(this.getClass(),
233  "HashDbIngestModule.lookingUpKnownBadHashValueErr",
234  name)));
235  ret = ProcessResult.ERROR;
236  }
237  }
238 
239  // If the file is not in the known bad sets, search for it in the known sets.
240  // Any hit is sufficient to classify it as known, and there is no need to create
241  // a hit artifact or send a message to the application inbox.
242  if (!foundBad) {
243  for (HashDb db : knownHashSets) {
244  try {
245  long lookupstart = System.currentTimeMillis();
246  if (db.lookupMD5Quick(file)) {
247  try {
248  skCase.setKnown(file, TskData.FileKnown.KNOWN);
249  break;
250  } catch (TskException ex) {
251  logger.log(Level.WARNING, "Couldn't set known state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
252  ret = ProcessResult.ERROR;
253  }
254  }
255  long delta = (System.currentTimeMillis() - lookupstart);
256  totals.totalLookuptime.addAndGet(delta);
257 
258  } catch (TskException ex) {
259  logger.log(Level.WARNING, "Couldn't lookup known hash for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
261  HashLookupModuleFactory.getModuleName(),
262  NbBundle.getMessage(this.getClass(),
263  "HashDbIngestModule.hashLookupErrorMsg",
264  name),
265  NbBundle.getMessage(this.getClass(),
266  "HashDbIngestModule.lookingUpKnownHashValueErr",
267  name)));
268  ret = ProcessResult.ERROR;
269  }
270  }
271  }
272 
273  return ret;
274  }
275 
276  private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) {
277  try {
278  String MODULE_NAME = NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.moduleName");
279 
281  //TODO Revisit usage of deprecated constructor as per TSK-583
282  //BlackboardAttribute att2 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), MODULE_NAME, "Known Bad", hashSetName);
283  BlackboardAttribute att2 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), MODULE_NAME, hashSetName);
284  badFile.addAttribute(att2);
285  BlackboardAttribute att3 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5.getTypeID(), MODULE_NAME, md5Hash);
286  badFile.addAttribute(att3);
287  BlackboardAttribute att4 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT.getTypeID(), MODULE_NAME, comment);
288  badFile.addAttribute(att4);
289 
290  if (showInboxMessage) {
291  StringBuilder detailsSb = new StringBuilder();
292  //details
293  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
294  //hit
295  detailsSb.append("<tr>"); //NON-NLS
296  detailsSb.append("<th>") //NON-NLS
297  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.fileName"))
298  .append("</th>"); //NON-NLS
299  detailsSb.append("<td>") //NON-NLS
300  .append(abstractFile.getName())
301  .append("</td>"); //NON-NLS
302  detailsSb.append("</tr>"); //NON-NLS
303 
304  detailsSb.append("<tr>"); //NON-NLS
305  detailsSb.append("<th>") //NON-NLS
306  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.md5Hash"))
307  .append("</th>"); //NON-NLS
308  detailsSb.append("<td>").append(md5Hash).append("</td>"); //NON-NLS
309  detailsSb.append("</tr>"); //NON-NLS
310 
311  detailsSb.append("<tr>"); //NON-NLS
312  detailsSb.append("<th>") //NON-NLS
313  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.hashsetName"))
314  .append("</th>"); //NON-NLS
315  detailsSb.append("<td>").append(hashSetName).append("</td>"); //NON-NLS
316  detailsSb.append("</tr>"); //NON-NLS
317 
318  detailsSb.append("</table>"); //NON-NLS
319 
321  NbBundle.getMessage(this.getClass(),
322  "HashDbIngestModule.postToBB.knownBadMsg",
323  abstractFile.getName()),
324  detailsSb.toString(),
325  abstractFile.getName() + md5Hash,
326  badFile));
327  }
328  services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_HASHSET_HIT, Collections.singletonList(badFile)));
329  } catch (TskException ex) {
330  logger.log(Level.WARNING, "Error creating blackboard artifact", ex); //NON-NLS
331  }
332  }
333 
334  private synchronized void postSummary() {
335  IngestJobTotals jobTotals = totalsForIngestJobs.remove(jobId);
336 
337  if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) {
338  StringBuilder detailsSb = new StringBuilder();
339  //details
340  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
341 
342  detailsSb.append("<tr><td>") //NON-NLS
343  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.knownBadsFound"))
344  .append("</td>"); //NON-NLS
345  detailsSb.append("<td>").append(jobTotals.totalKnownBadCount.get()).append("</td></tr>"); //NON-NLS
346 
347  detailsSb.append("<tr><td>") //NON-NLS
348  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.totalCalcTime"))
349  .append("</td><td>").append(jobTotals.totalCalctime.get()).append("</td></tr>\n"); //NON-NLS
350  detailsSb.append("<tr><td>") //NON-NLS
351  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.totalLookupTime"))
352  .append("</td><td>").append(jobTotals.totalLookuptime.get()).append("</td></tr>\n"); //NON-NLS
353  detailsSb.append("</table>"); //NON-NLS
354 
355  detailsSb.append("<p>") //NON-NLS
356  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.databasesUsed"))
357  .append("</p>\n<ul>"); //NON-NLS
358  for (HashDb db : knownBadHashSets) {
359  detailsSb.append("<li>").append(db.getHashSetName()).append("</li>\n"); //NON-NLS
360  }
361 
362  detailsSb.append("</ul>"); //NON-NLS
363 
366  HashLookupModuleFactory.getModuleName(),
367  NbBundle.getMessage(this.getClass(),
368  "HashDbIngestModule.complete.hashLookupResults"),
369  detailsSb.toString()));
370  }
371  }
372 
373  @Override
374  public void shutDown() {
375  if (refCounter.decrementAndGet(jobId) == 0) {
376  postSummary();
377  }
378  }
379 }
static String calculateMd5(AbstractFile file)
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
TskData.TSK_DB_FILES_TYPE_ENUM getType()
void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context)
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId)
void updateEnabledHashSets(List< HashDb > allHashSets, List< HashDb > enabledHashSets)
void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage)
static final HashMap< Long, IngestJobTotals > totalsForIngestJobs
void addAttribute(BlackboardAttribute attr)
void postMessage(final IngestMessage message)
void fireModuleDataEvent(ModuleDataEvent moduleDataEvent)
BlackboardArtifact newArtifact(int artifactTypeID)
boolean setKnown(AbstractFile file, FileKnown fileKnown)
static IngestMessage createWarningMessage(String source, String subject, String detailsHtml)
static Logger getLogger(String name)
Definition: Logger.java:131
static synchronized IngestServices getInstance()

Copyright © 2012-2015 Basis Technology. Generated on: Mon Oct 19 2015
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.