Autopsy  4.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
LuceneQuery.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2017 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import java.util.ArrayList;
22 import java.util.Collection;
23 import java.util.Comparator;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.logging.Level;
27 import org.apache.commons.lang3.StringUtils;
28 import org.apache.commons.lang3.math.NumberUtils;
29 import org.apache.solr.client.solrj.SolrQuery;
30 import org.apache.solr.client.solrj.SolrRequest;
31 import org.apache.solr.client.solrj.SolrRequest.METHOD;
32 import org.apache.solr.client.solrj.response.QueryResponse;
33 import org.apache.solr.common.SolrDocument;
34 import org.apache.solr.common.SolrDocumentList;
35 import org.apache.solr.common.params.CursorMarkParams;
45 
50 class LuceneQuery implements KeywordSearchQuery {
51 
52  private static final Logger logger = Logger.getLogger(LuceneQuery.class.getName());
53  private String keywordStringEscaped;
54  private boolean isEscaped;
55  private final Keyword originalKeyword ;
56  private final KeywordList keywordList ;
57  private final List<KeywordQueryFilter> filters = new ArrayList<>();
58  private String field = null;
59  private static final int MAX_RESULTS_PER_CURSOR_MARK = 512;
60  static final int SNIPPET_LENGTH = 50;
61  static final String HIGHLIGHT_FIELD = Server.Schema.TEXT.toString();
62 
63  private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);
64 
70  LuceneQuery(KeywordList keywordList, Keyword keyword) {
71  this.keywordList = keywordList;
72  this.originalKeyword = keyword;
73  this.keywordStringEscaped = this.originalKeyword.getSearchTerm();
74  }
75 
76  @Override
77  public void addFilter(KeywordQueryFilter filter) {
78  this.filters.add(filter);
79  }
80 
81  @Override
82  public void setField(String field) {
83  this.field = field;
84  }
85 
86  @Override
87  public void setSubstringQuery() {
88  // Note that this is not a full substring search. Normally substring
89  // searches will be done with TermComponentQuery objects instead.
90  keywordStringEscaped += "*";
91  }
92 
93  @Override
94  public void escape() {
95  keywordStringEscaped = KeywordSearchUtil.escapeLuceneQuery(originalKeyword.getSearchTerm());
96  isEscaped = true;
97  }
98 
99  @Override
100  public boolean isEscaped() {
101  return isEscaped;
102  }
103 
104  @Override
105  public boolean isLiteral() {
106  return originalKeyword.searchTermIsLiteral();
107  }
108 
109  @Override
110  public String getEscapedQueryString() {
111  return this.keywordStringEscaped;
112  }
113 
114  @Override
115  public String getQueryString() {
116  return this.originalKeyword.getSearchTerm();
117  }
118 
119  @Override
120  public KeywordList getKeywordList() {
121  return keywordList;
122  }
123 
124  @Override
125  public QueryResults performQuery() throws KeywordSearchModuleException, NoOpenCoreException {
126 
127  final Server solrServer = KeywordSearch.getServer();
128  double indexSchemaVersion = NumberUtils.toDouble(solrServer.getIndexInfo().getSchemaVersion());
129 
130  SolrQuery solrQuery = createAndConfigureSolrQuery(KeywordSearchSettings.getShowSnippets());
131 
132  final String strippedQueryString = StringUtils.strip(getQueryString(), "\"");
133 
134  String cursorMark = CursorMarkParams.CURSOR_MARK_START;
135  boolean allResultsProcessed = false;
136  List<KeywordHit> matches = new ArrayList<>();
137  while (!allResultsProcessed) {
138  solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
139  QueryResponse response = solrServer.query(solrQuery, SolrRequest.METHOD.POST);
140  SolrDocumentList resultList = response.getResults();
141  // objectId_chunk -> "text" -> List of previews
142  Map<String, Map<String, List<String>>> highlightResponse = response.getHighlighting();
143 
144  for (SolrDocument resultDoc : resultList) {
145  try {
146  /*
147  * for each result doc, check that the first occurence of
148  * that term is before the window. if all the ocurences
149  * start within the window, don't record them for this
150  * chunk, they will get picked up in the next one.
151  */
152  final String docId = resultDoc.getFieldValue(Server.Schema.ID.toString()).toString();
153  final Integer chunkSize = (Integer) resultDoc.getFieldValue(Server.Schema.CHUNK_SIZE.toString());
154  final Collection<Object> content = resultDoc.getFieldValues(Server.Schema.CONTENT_STR.toString());
155 
156  if (indexSchemaVersion < 2.0) {
157  //old schema versions don't support chunk_size or the content_str fields, so just accept hits
158  matches.add(createKeywordtHit(highlightResponse, docId));
159  } else {
160  //check against file name and actual content seperately.
161  for (Object content_obj : content) {
162  String content_str = (String) content_obj;
163  //for new schemas, check that the hit is before the chunk/window boundary.
164  int firstOccurence = StringUtils.indexOfIgnoreCase(content_str, strippedQueryString);
165  //there is no chunksize field for "parent" entries in the index
166  if (chunkSize == null || chunkSize == 0 || (firstOccurence > -1 && firstOccurence < chunkSize)) {
167  matches.add(createKeywordtHit(highlightResponse, docId));
168  }
169  }
170  }
171  } catch (TskException ex) {
172  throw new KeywordSearchModuleException(ex);
173  }
174  }
175  String nextCursorMark = response.getNextCursorMark();
176  if (cursorMark.equals(nextCursorMark)) {
177  allResultsProcessed = true;
178  }
179  cursorMark = nextCursorMark;
180  }
181 
182  QueryResults results = new QueryResults(this);
183  //in case of single term literal query there is only 1 term
184  results.addResult(new Keyword(originalKeyword.getSearchTerm(), true, true, originalKeyword.getListName(), originalKeyword.getOriginalTerm()), matches);
185 
186  return results;
187  }
188 
189  @Override
190  public boolean validate() {
191  return StringUtils.isNotBlank(originalKeyword.getSearchTerm());
192  }
193 
194  @Override
195  public KeywordCachedArtifact writeSingleFileHitsToBlackBoard(Keyword foundKeyword, KeywordHit hit, String snippet, String listName) {
196  final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName();
197 
198  Collection<BlackboardAttribute> attributes = new ArrayList<>();
199  BlackboardArtifact bba;
200  KeywordCachedArtifact writeResult;
201  try {
202  bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
203  writeResult = new KeywordCachedArtifact(bba);
204  } catch (TskCoreException e) {
205  logger.log(Level.WARNING, "Error adding bb artifact for keyword hit", e); //NON-NLS
206  return null;
207  }
208 
209  if (snippet != null) {
210  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet));
211  }
212  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm()));
213  if (StringUtils.isNotBlank(listName)) {
214  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName));
215  }
216 
217  if (originalKeyword != null) {
218  BlackboardAttribute.ATTRIBUTE_TYPE selType = originalKeyword.getArtifactAttributeType();
219  if (selType != null) {
220  attributes.add(new BlackboardAttribute(selType, MODULE_NAME, foundKeyword.getSearchTerm()));
221  }
222 
223  if (originalKeyword.searchTermIsWholeWord()) {
224  attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.LITERAL.ordinal()));
225  } else {
226  attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal()));
227  }
228  }
229 
230  if (hit.isArtifactHit()) {
231  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, hit.getArtifact().getArtifactID()));
232  }
233 
234  try {
235  bba.addAttributes(attributes); //write out to bb
236  writeResult.add(attributes);
237  return writeResult;
238  } catch (TskCoreException e) {
239  logger.log(Level.WARNING, "Error adding bb attributes to artifact", e); //NON-NLS
240  return null;
241  }
242  }
243 
244 
245  /*
246  * Create the query object for the stored keyword
247  *
248  * @param snippets True if query should request snippets
249  *
250  * @return
251  */
252  private SolrQuery createAndConfigureSolrQuery(boolean snippets) {
253  SolrQuery q = new SolrQuery();
254  q.setShowDebugInfo(DEBUG); //debug
255  // Wrap the query string in quotes if this is a literal search term.
256  String queryStr = originalKeyword.searchTermIsLiteral()
257  ? KeywordSearchUtil.quoteQuery(keywordStringEscaped) : keywordStringEscaped;
258 
259  // Run the query against an optional alternative field.
260  if (field != null) {
261  //use the optional field
262  queryStr = field + ":" + queryStr;
263  }
264  q.setQuery(queryStr);
265  q.setRows(MAX_RESULTS_PER_CURSOR_MARK);
266  // Setting the sort order is necessary for cursor based paging to work.
267  q.setSort(SolrQuery.SortClause.asc(Server.Schema.ID.toString()));
268 
269  q.setFields(Server.Schema.ID.toString(),
270  Server.Schema.CHUNK_SIZE.toString(),
271  Server.Schema.CONTENT_STR.toString());
272 
273  for (KeywordQueryFilter filter : filters) {
274  q.addFilterQuery(filter.toString());
275  }
276 
277  if (snippets) {
278  configurwQueryForHighlighting(q);
279  }
280 
281  return q;
282  }
283 
290  private static void configurwQueryForHighlighting(SolrQuery q) {
291  q.addHighlightField(HIGHLIGHT_FIELD);
292  q.setHighlightSnippets(1);
293  q.setHighlightFragsize(SNIPPET_LENGTH);
294 
295  //tune the highlighter
296  q.setParam("hl.useFastVectorHighlighter", "on"); //fast highlighter scales better than standard one NON-NLS
297  q.setParam("hl.tag.pre", "&laquo;"); //makes sense for FastVectorHighlighter only NON-NLS
298  q.setParam("hl.tag.post", "&laquo;"); //makes sense for FastVectorHighlighter only NON-NLS
299  q.setParam("hl.fragListBuilder", "simple"); //makes sense for FastVectorHighlighter only NON-NLS
300 
301  //Solr bug if fragCharSize is smaller than Query string, StringIndexOutOfBoundsException is thrown.
302  q.setParam("hl.fragCharSize", Integer.toString(q.getQuery().length())); //makes sense for FastVectorHighlighter only NON-NLS
303 
304  //docs says makes sense for the original Highlighter only, but not really
305  //analyze all content SLOW! consider lowering
306  q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED); //NON-NLS
307  }
308 
309  private KeywordHit createKeywordtHit(Map<String, Map<String, List<String>>> highlightResponse, String docId) throws TskException {
314  String snippet = "";
315  if (KeywordSearchSettings.getShowSnippets()) {
316  List<String> snippetList = highlightResponse.get(docId).get(Server.Schema.TEXT.toString());
317  // list is null if there wasn't a snippet
318  if (snippetList != null) {
319  snippet = EscapeUtil.unEscapeHtml(snippetList.get(0)).trim();
320  }
321  }
322 
323  return new KeywordHit(docId, snippet, originalKeyword.getSearchTerm());
324  }
325 
340  static String querySnippet(String query, long solrObjectId, boolean isRegex, boolean group) throws NoOpenCoreException {
341  return querySnippet(query, solrObjectId, 0, isRegex, group);
342  }
343 
359  static String querySnippet(String query, long solrObjectId, int chunkID, boolean isRegex, boolean group) throws NoOpenCoreException {
360  SolrQuery q = new SolrQuery();
361  q.setShowDebugInfo(DEBUG); //debug
362 
363  String queryStr;
364  if (isRegex) {
365  queryStr = HIGHLIGHT_FIELD + ":"
366  + (group ? KeywordSearchUtil.quoteQuery(query)
367  : query);
368  } else {
369  /*
370  * simplify query/escaping and use default field always force
371  * grouping/quotes
372  */
373  queryStr = KeywordSearchUtil.quoteQuery(query);
374  }
375  q.setQuery(queryStr);
376 
377  String contentIDStr = (chunkID == 0)
378  ? Long.toString(solrObjectId)
379  : Server.getChunkIdString(solrObjectId, chunkID);
380  String idQuery = Server.Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(contentIDStr);
381  q.addFilterQuery(idQuery);
382 
383  configurwQueryForHighlighting(q);
384 
385  Server solrServer = KeywordSearch.getServer();
386 
387  try {
388  QueryResponse response = solrServer.query(q, METHOD.POST);
389  Map<String, Map<String, List<String>>> responseHighlight = response.getHighlighting();
390  Map<String, List<String>> responseHighlightID = responseHighlight.get(contentIDStr);
391  if (responseHighlightID == null) {
392  return "";
393  }
394  List<String> contentHighlights = responseHighlightID.get(LuceneQuery.HIGHLIGHT_FIELD);
395  if (contentHighlights == null) {
396  return "";
397  } else {
398  // extracted content is HTML-escaped, but snippet goes in a plain text field
399  return EscapeUtil.unEscapeHtml(contentHighlights.get(0)).trim();
400  }
401  } catch (NoOpenCoreException ex) {
402  logger.log(Level.WARNING, "Error executing Lucene Solr Query: " + query, ex); //NON-NLS
403  throw ex;
404  } catch (KeywordSearchModuleException ex) {
405  logger.log(Level.WARNING, "Error executing Lucene Solr Query: " + query, ex); //NON-NLS
406  return "";
407  }
408  }
409 }

Copyright © 2012-2016 Basis Technology. Generated on: Mon Apr 24 2017
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.