Autopsy  4.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
SearchRunner.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011 - 2014 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import java.util.ArrayList;
22 import java.util.Collection;
23 import java.util.HashMap;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.Map.Entry;
27 import java.util.Timer;
28 import java.util.TimerTask;
29 import java.util.concurrent.CancellationException;
30 import java.util.concurrent.ExecutionException;
31 import java.util.concurrent.atomic.AtomicLong;
32 import java.util.logging.Level;
33 import javax.swing.SwingUtilities;
34 import javax.swing.SwingWorker;
35 import org.netbeans.api.progress.aggregate.AggregateProgressFactory;
36 import org.netbeans.api.progress.aggregate.AggregateProgressHandle;
37 import org.netbeans.api.progress.aggregate.ProgressContributor;
38 import org.openide.util.Cancellable;
39 import org.openide.util.NbBundle;
40 import org.openide.util.NbBundle.Messages;
46 import org.sleuthkit.datamodel.BlackboardArtifact;
47 
52 public final class SearchRunner {
53 
54  private static final Logger logger = Logger.getLogger(SearchRunner.class.getName());
55  private static SearchRunner instance = null;
57  private Ingester ingester = null;
58  private volatile boolean updateTimerRunning = false;
59  private Timer updateTimer;
60 
61  // maps a jobID to the search
62  private Map<Long, SearchJobInfo> jobs = new HashMap<>(); //guarded by "this"
63 
64  SearchRunner() {
65  ingester = Server.getIngester();
66  updateTimer = new Timer(NbBundle.getMessage(this.getClass(), "SearchRunner.updateTimer.title.text"), true); // run as a daemon
67  }
68 
73  public static synchronized SearchRunner getInstance() {
74  if (instance == null) {
75  instance = new SearchRunner();
76  }
77  return instance;
78  }
79 
90  public synchronized void startJob(long jobId, long dataSourceId, List<String> keywordListNames) {
91  if (jobs.containsKey(jobId) == false) {
92  logger.log(Level.INFO, "Adding job {0}", jobId); //NON-NLS
93  SearchJobInfo jobData = new SearchJobInfo(jobId, dataSourceId, keywordListNames);
94  jobs.put(jobId, jobData);
95  }
96 
97  // keep track of how many threads / module instances from this job have asked for this
98  jobs.get(jobId).incrementModuleReferenceCount();
99 
100  // start the timer, if needed
101  if ((jobs.size() > 0) && (updateTimerRunning == false)) {
102  final long updateIntervalMs = ((long) KeywordSearchSettings.getUpdateFrequency().getTime()) * 60 * 1000;
103  updateTimer.scheduleAtFixedRate(new UpdateTimerTask(), updateIntervalMs, updateIntervalMs);
104  updateTimerRunning = true;
105  }
106  }
107 
114  public void endJob(long jobId) {
115  SearchJobInfo job;
116  boolean readyForFinalSearch = false;
117  synchronized (this) {
118  job = jobs.get(jobId);
119  if (job == null) {
120  return;
121  }
122 
123  // Only do final search if this is the last module/thread in this job to call endJob()
124  if (job.decrementModuleReferenceCount() == 0) {
125  jobs.remove(jobId);
126  readyForFinalSearch = true;
127  }
128  }
129 
130  if (readyForFinalSearch) {
131  commit();
132  doFinalSearch(job); //this will block until it's done
133  }
134  }
135 
142  public void stopJob(long jobId) {
143  logger.log(Level.INFO, "Stopping job {0}", jobId); //NON-NLS
144  commit();
145 
146  SearchJobInfo job;
147  synchronized (this) {
148  job = jobs.get(jobId);
149  if (job == null) {
150  return;
151  }
152 
153  //stop currentSearcher
154  SearchRunner.Searcher currentSearcher = job.getCurrentSearcher();
155  if ((currentSearcher != null) && (!currentSearcher.isDone())) {
156  currentSearcher.cancel(true);
157  }
158 
159  jobs.remove(jobId);
160  }
161  }
162 
169  public synchronized void addKeywordListsToAllJobs(List<String> keywordListNames) {
170  for (String listName : keywordListNames) {
171  logger.log(Level.INFO, "Adding keyword list {0} to all jobs", listName); //NON-NLS
172  for (SearchJobInfo j : jobs.values()) {
173  j.addKeywordListName(listName);
174  }
175  }
176  }
177 
181  private void commit() {
182  ingester.commit();
183 
184  // Signal a potential change in number of text_ingested files
185  try {
186  final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles();
187  KeywordSearch.fireNumIndexedFilesChange(null, numIndexedFiles);
189  logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", ex); //NON-NLS
190  }
191  }
192 
199  private void doFinalSearch(SearchJobInfo job) {
200  // Run one last search as there are probably some new files committed
201  logger.log(Level.INFO, "Running final search for jobid {0}", job.getJobId()); //NON-NLS
202  if (!job.getKeywordListNames().isEmpty()) {
203  try {
204  // In case this job still has a worker running, wait for it to finish
205  job.waitForCurrentWorker();
206 
207  SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job, true);
208  job.setCurrentSearcher(finalSearcher); //save the ref
209  finalSearcher.execute(); //start thread
210 
211  // block until the search is complete
212  finalSearcher.get();
213 
214  } catch (InterruptedException | ExecutionException ex) {
215  logger.log(Level.WARNING, "Job {1} final search thread failed: {2}", new Object[]{job.getJobId(), ex}); //NON-NLS
216  }
217  }
218  }
219 
223  private class UpdateTimerTask extends TimerTask {
224 
225  private final Logger logger = Logger.getLogger(SearchRunner.UpdateTimerTask.class.getName());
226 
227  @Override
228  public void run() {
229  // If no jobs then cancel the task. If more job(s) come along, a new task will start up.
230  if (jobs.isEmpty()) {
231  this.cancel(); //terminate this timer task
232  updateTimerRunning = false;
233  return;
234  }
235 
236  commit();
237 
238  synchronized (SearchRunner.this) {
239  // Spawn a search thread for each job
240  for (Entry<Long, SearchJobInfo> j : jobs.entrySet()) {
241  SearchJobInfo job = j.getValue();
242  // If no lists or the worker is already running then skip it
243  if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) {
244  Searcher searcher = new Searcher(job);
245  job.setCurrentSearcher(searcher); //save the ref
246  searcher.execute(); //start thread
247  job.setWorkerRunning(true);
248  }
249  }
250  }
251  }
252  }
253 
258  private class SearchJobInfo {
259 
260  private final long jobId;
261  private final long dataSourceId;
262  // mutable state:
263  private volatile boolean workerRunning;
264  private List<String> keywordListNames; //guarded by SearchJobInfo.this
265  private Map<Keyword, List<Long>> currentResults; //guarded by SearchJobInfo.this
267  private AtomicLong moduleReferenceCount = new AtomicLong(0);
268  private final Object finalSearchLock = new Object(); //used for a condition wait
269 
270  public SearchJobInfo(long jobId, long dataSourceId, List<String> keywordListNames) {
271  this.jobId = jobId;
272  this.dataSourceId = dataSourceId;
273  this.keywordListNames = new ArrayList<>(keywordListNames);
274  currentResults = new HashMap<>();
275  workerRunning = false;
276  currentSearcher = null;
277  }
278 
279  public long getJobId() {
280  return jobId;
281  }
282 
283  public long getDataSourceId() {
284  return dataSourceId;
285  }
286 
287  public synchronized List<String> getKeywordListNames() {
288  return new ArrayList<>(keywordListNames);
289  }
290 
291  public synchronized void addKeywordListName(String keywordListName) {
292  if (!keywordListNames.contains(keywordListName)) {
293  keywordListNames.add(keywordListName);
294  }
295  }
296 
297  public synchronized List<Long> currentKeywordResults(Keyword k) {
298  return currentResults.get(k);
299  }
300 
301  public synchronized void addKeywordResults(Keyword k, List<Long> resultsIDs) {
302  currentResults.put(k, resultsIDs);
303  }
304 
305  public boolean isWorkerRunning() {
306  return workerRunning;
307  }
308 
309  public void setWorkerRunning(boolean flag) {
310  workerRunning = flag;
311  }
312 
313  public synchronized SearchRunner.Searcher getCurrentSearcher() {
314  return currentSearcher;
315  }
316 
317  public synchronized void setCurrentSearcher(SearchRunner.Searcher searchRunner) {
318  currentSearcher = searchRunner;
319  }
320 
322  moduleReferenceCount.incrementAndGet();
323  }
324 
326  return moduleReferenceCount.decrementAndGet();
327  }
328 
334  public void waitForCurrentWorker() throws InterruptedException {
335  synchronized (finalSearchLock) {
336  while (workerRunning) {
337  finalSearchLock.wait(); //wait() releases the lock
338  }
339  }
340  }
341 
345  public void searchNotify() {
346  synchronized (finalSearchLock) {
347  workerRunning = false;
348  finalSearchLock.notify();
349  }
350  }
351  }
352 
359  private final class Searcher extends SwingWorker<Object, Void> {
360 
365  private List<Keyword> keywords; //keywords to search
366  private List<String> keywordListNames; // lists currently being searched
367  private List<KeywordList> keywordLists;
368  private Map<String, KeywordList> keywordToList; //keyword to list name mapping
369  private AggregateProgressHandle progressGroup;
370  private final Logger logger = Logger.getLogger(SearchRunner.Searcher.class.getName());
371  private boolean finalRun = false;
372 
373  Searcher(SearchJobInfo job) {
374  this.job = job;
375  keywordListNames = job.getKeywordListNames();
376  keywords = new ArrayList<>();
377  keywordToList = new HashMap<>();
378  keywordLists = new ArrayList<>();
379  //keywords are populated as searcher runs
380  }
381 
382  Searcher(SearchJobInfo job, boolean finalRun) {
383  this(job);
384  this.finalRun = finalRun;
385  }
386 
387  @Override
388  @Messages("SearchRunner.query.exception.msg=Error performing query:")
389  protected Object doInBackground() throws Exception {
390  final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName")
391  + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : "");
392  final String pgDisplayName = displayName + (" (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")");
393  progressGroup = AggregateProgressFactory.createSystemHandle(pgDisplayName, null, new Cancellable() {
394  @Override
395  public boolean cancel() {
396  logger.log(Level.INFO, "Cancelling the searcher by user."); //NON-NLS
397  if (progressGroup != null) {
398  progressGroup.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg"));
399  }
400  return SearchRunner.Searcher.this.cancel(true);
401  }
402  }, null);
403 
404  updateKeywords();
405 
406  ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()];
407  int i = 0;
408  for (Keyword keywordQuery : keywords) {
409  subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getSearchTerm());
410  progressGroup.addContributor(subProgresses[i]);
411  i++;
412  }
413 
414  progressGroup.start();
415 
416  final StopWatch stopWatch = new StopWatch();
417  stopWatch.start();
418  try {
419  progressGroup.setDisplayName(displayName);
420 
421  int keywordsSearched = 0;
422 
423  for (Keyword keywordQuery : keywords) {
424  if (this.isCancelled()) {
425  logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keywordQuery.getSearchTerm()); //NON-NLS
426  return null;
427  }
428 
429  final String queryStr = keywordQuery.getSearchTerm();
430  final KeywordList list = keywordToList.get(queryStr);
431 
432  //new subProgress will be active after the initial query
433  //when we know number of hits to start() with
434  if (keywordsSearched > 0) {
435  subProgresses[keywordsSearched - 1].finish();
436  }
437 
438  KeywordSearchQuery keywordSearchQuery = null;
439 
440  boolean isRegex = !keywordQuery.searchTermIsLiteral();
441  if (isRegex) {
442  keywordSearchQuery = new TermsComponentQuery(list, keywordQuery);
443  } else {
444  keywordSearchQuery = new LuceneQuery(list, keywordQuery);
445  keywordSearchQuery.escape();
446  }
447 
448  // Filtering
449  //limit search to currently ingested data sources
450  //set up a filter with 1 or more image ids OR'ed
451  final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId());
452  keywordSearchQuery.addFilter(dataSourceFilter);
453 
454  QueryResults queryResults;
455 
456  // Do the actual search
457  try {
458  queryResults = keywordSearchQuery.performQuery();
460  logger.log(Level.SEVERE, "Error performing query: " + keywordQuery.getSearchTerm(), ex); //NON-NLS
461  MessageNotifyUtil.Notify.error(Bundle.SearchRunner_query_exception_msg() + keywordQuery.getSearchTerm(), ex.getCause().getMessage());
462  //no reason to continue with next query if recovery failed
463  //or wait for recovery to kick in and run again later
464  //likely case has closed and threads are being interrupted
465  return null;
466  } catch (CancellationException e) {
467  logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keywordQuery.getSearchTerm()); //NON-NLS
468  return null;
469  }
470 
471  // calculate new results by substracting results already obtained in this ingest
472  // this creates a map of each keyword to the list of unique files that have that hit.
473  QueryResults newResults = filterResults(queryResults);
474 
475  if (!newResults.getKeywords().isEmpty()) {
476 
477  // Write results to BB
478  //new artifacts created, to report to listeners
479  Collection<BlackboardArtifact> newArtifacts = new ArrayList<>();
480 
481  //scale progress bar more more granular, per result sub-progress, within per keyword
482  int totalUnits = newResults.getKeywords().size();
483  subProgresses[keywordsSearched].start(totalUnits);
484  int unitProgress = 0;
485  String queryDisplayStr = keywordQuery.getSearchTerm();
486  if (queryDisplayStr.length() > 50) {
487  queryDisplayStr = queryDisplayStr.substring(0, 49) + "...";
488  }
489  subProgresses[keywordsSearched].progress(list.getName() + ": " + queryDisplayStr, unitProgress);
490 
491  // Create blackboard artifacts
492  newArtifacts = newResults.writeAllHitsToBlackBoard(null, subProgresses[keywordsSearched], this, list.getIngestMessages());
493 
494  } //if has results
495 
496  //reset the status text before it goes away
497  subProgresses[keywordsSearched].progress("");
498 
499  ++keywordsSearched;
500 
501  } //for each keyword
502 
503  } //end try block
504  catch (Exception ex) {
505  logger.log(Level.WARNING, "searcher exception occurred", ex); //NON-NLS
506  } finally {
507  try {
509  stopWatch.stop();
510 
511  logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); //NON-NLS
512  } finally {
513  // In case a thread is waiting on this worker to be done
514  job.searchNotify();
515  }
516  }
517 
518  return null;
519  }
520 
521  @Override
522  protected void done() {
523  // call get to see if there were any errors
524  try {
525  get();
526  } catch (InterruptedException | ExecutionException e) {
527  logger.log(Level.SEVERE, "Error performing keyword search: " + e.getMessage()); //NON-NLS
529  NbBundle.getMessage(this.getClass(),
530  "SearchRunner.Searcher.done.err.msg"), e.getMessage()));
531  } // catch and ignore if we were cancelled
532  catch (java.util.concurrent.CancellationException ex) {
533  }
534  }
535 
539  private void updateKeywords() {
540  XmlKeywordSearchList loader = XmlKeywordSearchList.getCurrent();
541 
542  keywords.clear();
543  keywordToList.clear();
544  keywordLists.clear();
545 
546  for (String name : keywordListNames) {
547  KeywordList list = loader.getList(name);
548  keywordLists.add(list);
549  for (Keyword k : list.getKeywords()) {
550  keywords.add(k);
551  keywordToList.put(k.getSearchTerm(), list);
552  }
553  }
554  }
555 
561  private void finalizeSearcher() {
562  SwingUtilities.invokeLater(new Runnable() {
563  @Override
564  public void run() {
565  progressGroup.finish();
566  }
567  });
568  }
569 
570  //calculate new results but substracting results already obtained in this ingest
571  //update currentResults map with the new results
572  private QueryResults filterResults(QueryResults queryResult) {
573 
574  QueryResults newResults = new QueryResults(queryResult.getQuery(), queryResult.getKeywordList());
575 
576  for (Keyword keyword : queryResult.getKeywords()) {
577  List<KeywordHit> queryTermResults = queryResult.getResults(keyword);
578 
579  //translate to list of IDs that we keep track of
580  List<Long> queryTermResultsIDs = new ArrayList<>();
581  for (KeywordHit ch : queryTermResults) {
582  queryTermResultsIDs.add(ch.getSolrObjectId());
583  }
584 
585  List<Long> curTermResults = job.currentKeywordResults(keyword);
586  if (curTermResults == null) {
587  job.addKeywordResults(keyword, queryTermResultsIDs);
588  newResults.addResult(keyword, queryTermResults);
589  } else {
590  //some AbstractFile hits already exist for this keyword
591  for (KeywordHit res : queryTermResults) {
592  if (!curTermResults.contains(res.getSolrObjectId())) {
593  //add to new results
594  List<KeywordHit> newResultsFs = newResults.getResults(keyword);
595  if (newResultsFs == null) {
596  newResultsFs = new ArrayList<>();
597  newResults.addResult(keyword, newResultsFs);
598  }
599  newResultsFs.add(res);
600  curTermResults.add(res.getSolrObjectId());
601  }
602  }
603  }
604  }
605 
606  return newResults;
607  }
608  }
609 }
SearchJobInfo(long jobId, long dataSourceId, List< String > keywordListNames)
synchronized void addKeywordResults(Keyword k, List< Long > resultsIDs)
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
static void fireNumIndexedFilesChange(Integer oldNum, Integer newNum)
synchronized void addKeywordListName(String keywordListName)
synchronized void startJob(long jobId, long dataSourceId, List< String > keywordListNames)
synchronized void setCurrentSearcher(SearchRunner.Searcher searchRunner)
synchronized List< Long > currentKeywordResults(Keyword k)
static synchronized SearchRunner getInstance()
synchronized void addKeywordListsToAllJobs(List< String > keywordListNames)
void postMessage(final IngestMessage message)
static void error(String title, String message)
synchronized static Logger getLogger(String name)
Definition: Logger.java:161
QueryResults filterResults(QueryResults queryResult)
static synchronized IngestServices getInstance()

Copyright © 2012-2016 Basis Technology. Generated on: Mon Jan 2 2017
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.